aboutsummaryrefslogtreecommitdiff
path: root/tokenizer.go
diff options
context:
space:
mode:
authorAlexander Kiryukhin <a.kiryukhin@mail.ru>2022-06-13 04:31:31 +0300
committerAlexander Kiryukhin <a.kiryukhin@mail.ru>2022-06-13 04:31:31 +0300
commit05626592208f56d88523887e2b80c0514f8ac210 (patch)
treeb0db34890e7366008754e975a8f28e878c5b28bf /tokenizer.go
initial
Diffstat (limited to 'tokenizer.go')
-rw-r--r--tokenizer.go93
1 files changed, 93 insertions, 0 deletions
diff --git a/tokenizer.go b/tokenizer.go
new file mode 100644
index 0000000..0d9ce08
--- /dev/null
+++ b/tokenizer.go
@@ -0,0 +1,93 @@
+package lexpr
+
+import (
+ "context"
+ "fmt"
+ "strconv"
+)
+
+func (l *Lexpr) tokenize(ctx context.Context, lexems <-chan lexem) <-chan Token {
+ out := make(chan Token)
+ go func() {
+ defer close(out)
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case lexem, ok := <-lexems:
+ if !ok {
+ return
+ }
+ switch {
+ case lexem.Type == lp:
+ out <- Token{
+ typ: lp,
+ }
+ case lexem.Type == rp:
+ out <- Token{
+ typ: rp,
+ }
+ case lexem.Type == sep:
+ out <- Token{
+ typ: sep,
+ }
+ case lexem.Type == number:
+ ivalue, _ := strconv.Atoi(lexem.Value)
+ out <- Token{
+ typ: number,
+ ivalue: ivalue,
+ }
+ case lexem.Type == str:
+ out <- Token{
+ typ: str,
+ value: lexem.Value,
+ }
+ case lexem.Type == op:
+ o, isOp := l.operators[lexem.Value]
+ if !isOp {
+ out <- Token{
+ typ: tokError,
+ value: fmt.Sprintf("unknown operator: %s", lexem.Value),
+ }
+ return
+ }
+ out <- Token{
+ typ: op,
+ value: lexem.Value,
+ priority: o.priority,
+ leftAssoc: o.leftAssoc,
+ }
+ case lexem.Type == word:
+ o, isOp := l.operators[lexem.Value]
+ _, isFunc := l.functions[lexem.Value]
+ switch {
+ case isOp:
+ out <- Token{
+ typ: op,
+ value: lexem.Value,
+ priority: o.priority,
+ leftAssoc: o.leftAssoc,
+ }
+ case isFunc:
+ out <- Token{
+ typ: funct,
+ value: lexem.Value,
+ }
+ default:
+ out <- Token{
+ typ: word,
+ value: lexem.Value,
+ }
+ }
+ case lexem.Type == tokError:
+ out <- Token{
+ typ: tokError,
+ value: lexem.Value,
+ }
+ return
+ }
+ }
+ }
+ }()
+ return out
+}