diff options
author | Alexander Kiryukhin <a.kiryukhin@mail.ru> | 2022-06-13 04:31:31 +0300 |
---|---|---|
committer | Alexander Kiryukhin <a.kiryukhin@mail.ru> | 2022-06-13 04:31:31 +0300 |
commit | 05626592208f56d88523887e2b80c0514f8ac210 (patch) | |
tree | b0db34890e7366008754e975a8f28e878c5b28bf /tokenizer.go |
initial
Diffstat (limited to 'tokenizer.go')
-rw-r--r-- | tokenizer.go | 93 |
1 files changed, 93 insertions, 0 deletions
diff --git a/tokenizer.go b/tokenizer.go new file mode 100644 index 0000000..0d9ce08 --- /dev/null +++ b/tokenizer.go @@ -0,0 +1,93 @@ +package lexpr + +import ( + "context" + "fmt" + "strconv" +) + +func (l *Lexpr) tokenize(ctx context.Context, lexems <-chan lexem) <-chan Token { + out := make(chan Token) + go func() { + defer close(out) + for { + select { + case <-ctx.Done(): + return + case lexem, ok := <-lexems: + if !ok { + return + } + switch { + case lexem.Type == lp: + out <- Token{ + typ: lp, + } + case lexem.Type == rp: + out <- Token{ + typ: rp, + } + case lexem.Type == sep: + out <- Token{ + typ: sep, + } + case lexem.Type == number: + ivalue, _ := strconv.Atoi(lexem.Value) + out <- Token{ + typ: number, + ivalue: ivalue, + } + case lexem.Type == str: + out <- Token{ + typ: str, + value: lexem.Value, + } + case lexem.Type == op: + o, isOp := l.operators[lexem.Value] + if !isOp { + out <- Token{ + typ: tokError, + value: fmt.Sprintf("unknown operator: %s", lexem.Value), + } + return + } + out <- Token{ + typ: op, + value: lexem.Value, + priority: o.priority, + leftAssoc: o.leftAssoc, + } + case lexem.Type == word: + o, isOp := l.operators[lexem.Value] + _, isFunc := l.functions[lexem.Value] + switch { + case isOp: + out <- Token{ + typ: op, + value: lexem.Value, + priority: o.priority, + leftAssoc: o.leftAssoc, + } + case isFunc: + out <- Token{ + typ: funct, + value: lexem.Value, + } + default: + out <- Token{ + typ: word, + value: lexem.Value, + } + } + case lexem.Type == tokError: + out <- Token{ + typ: tokError, + value: lexem.Value, + } + return + } + } + } + }() + return out +} |