12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576 |
- package chroma
-
- import "strings"
-
- // An Iterator across tokens.
- //
- // nil will be returned at the end of the Token stream.
- //
- // If an error occurs within an Iterator, it may propagate this in a panic. Formatters should recover.
- type Iterator func() Token
-
- // Tokens consumes all tokens from the iterator and returns them as a slice.
- func (i Iterator) Tokens() []Token {
- var out []Token
- for t := i(); t != EOF; t = i() {
- out = append(out, t)
- }
- return out
- }
-
- // Concaterator concatenates tokens from a series of iterators.
- func Concaterator(iterators ...Iterator) Iterator {
- return func() Token {
- for len(iterators) > 0 {
- t := iterators[0]()
- if t != EOF {
- return t
- }
- iterators = iterators[1:]
- }
- return EOF
- }
- }
-
- // Literator converts a sequence of literal Tokens into an Iterator.
- func Literator(tokens ...Token) Iterator {
- return func() Token {
- if len(tokens) == 0 {
- return EOF
- }
- token := tokens[0]
- tokens = tokens[1:]
- return token
- }
- }
-
- // SplitTokensIntoLines splits tokens containing newlines in two.
- func SplitTokensIntoLines(tokens []Token) (out [][]Token) {
- var line []Token // nolint: prealloc
- for _, token := range tokens {
- for strings.Contains(token.Value, "\n") {
- parts := strings.SplitAfterN(token.Value, "\n", 2)
- // Token becomes the tail.
- token.Value = parts[1]
-
- // Append the head to the line and flush the line.
- clone := token.Clone()
- clone.Value = parts[0]
- line = append(line, clone)
- out = append(out, line)
- line = nil
- }
- line = append(line, token)
- }
- if len(line) > 0 {
- out = append(out, line)
- }
- // Strip empty trailing token line.
- if len(out) > 0 {
- last := out[len(out)-1]
- if len(last) == 1 && last[0].Value == "" {
- out = out[:len(out)-1]
- }
- }
- return
- }
|