tokenize

Simlpe tokenize chain

Example:

package main

import t "github.com/rekki/go-query-analyze/tokenize"

func main() {
  tokenizer := []t.Tokenizer{
    t.NewWhitespace(),
    t.NewLeftEdge(1),
    t.NewUnique(),
  }
  tokens := t.Tokenize("hello world", tokenizer...)

  fmt.Printf("%v", tokens) // [h he hel hell hello w wo wor worl world]
}