You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
123456789101112131415 |
- // +build flex
-
- package tokenizer
-
- import "github.com/src-d/enry/v2/internal/tokenizer/flex"
-
- // Tokenize returns lexical tokens from content. The tokens returned match what
- // the Linguist library returns. At most the first ByteLimit bytes of content are tokenized.
- func Tokenize(content []byte) []string {
- if len(content) > ByteLimit {
- content = content[:ByteLimit]
- }
-
- return flex.TokenizeFlex(content)
- }
|