123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347 |
- package chroma
-
- import (
- "encoding/json"
- "fmt"
- )
-
- //go:generate stringer -type TokenType
-
- // TokenType is the type of token to highlight.
- //
- // It is also an Emitter, emitting a single token of itself
- type TokenType int
-
- func (t TokenType) MarshalJSON() ([]byte, error) { return json.Marshal(t.String()) }
- func (t *TokenType) UnmarshalJSON(data []byte) error {
- key := ""
- err := json.Unmarshal(data, &key)
- if err != nil {
- return err
- }
- for tt, text := range _TokenType_map {
- if text == key {
- *t = tt
- return nil
- }
- }
- return fmt.Errorf("unknown TokenType %q", data)
- }
-
- // Set of TokenTypes.
- //
- // Categories of types are grouped in ranges of 1000, while sub-categories are in ranges of 100. For
- // example, the literal category is in the range 3000-3999. The sub-category for literal strings is
- // in the range 3100-3199.
-
- // Meta token types.
- const (
- // Default background style.
- Background TokenType = -1 - iota
- // Line numbers in output.
- LineNumbers
- // Line numbers in output when in table.
- LineNumbersTable
- // Line higlight style.
- LineHighlight
- // Line numbers table wrapper style.
- LineTable
- // Line numbers table TD wrapper style.
- LineTableTD
- // Input that could not be tokenised.
- Error
- // Other is used by the Delegate lexer to indicate which tokens should be handled by the delegate.
- Other
- // No highlighting.
- None
- // Used as an EOF marker / nil token
- EOFType TokenType = 0
- )
-
- // Keywords.
- const (
- Keyword TokenType = 1000 + iota
- KeywordConstant
- KeywordDeclaration
- KeywordNamespace
- KeywordPseudo
- KeywordReserved
- KeywordType
- )
-
- // Names.
- const (
- Name TokenType = 2000 + iota
- NameAttribute
- NameBuiltin
- NameBuiltinPseudo
- NameClass
- NameConstant
- NameDecorator
- NameEntity
- NameException
- NameFunction
- NameFunctionMagic
- NameKeyword
- NameLabel
- NameNamespace
- NameOperator
- NameOther
- NamePseudo
- NameProperty
- NameTag
- NameVariable
- NameVariableAnonymous
- NameVariableClass
- NameVariableGlobal
- NameVariableInstance
- NameVariableMagic
- )
-
- // Literals.
- const (
- Literal TokenType = 3000 + iota
- LiteralDate
- LiteralOther
- )
-
- // Strings.
- const (
- LiteralString TokenType = 3100 + iota
- LiteralStringAffix
- LiteralStringAtom
- LiteralStringBacktick
- LiteralStringBoolean
- LiteralStringChar
- LiteralStringDelimiter
- LiteralStringDoc
- LiteralStringDouble
- LiteralStringEscape
- LiteralStringHeredoc
- LiteralStringInterpol
- LiteralStringName
- LiteralStringOther
- LiteralStringRegex
- LiteralStringSingle
- LiteralStringSymbol
- )
-
- // Literals.
- const (
- LiteralNumber TokenType = 3200 + iota
- LiteralNumberBin
- LiteralNumberFloat
- LiteralNumberHex
- LiteralNumberInteger
- LiteralNumberIntegerLong
- LiteralNumberOct
- )
-
- // Operators.
- const (
- Operator TokenType = 4000 + iota
- OperatorWord
- )
-
- // Punctuation.
- const (
- Punctuation TokenType = 5000 + iota
- )
-
- // Comments.
- const (
- Comment TokenType = 6000 + iota
- CommentHashbang
- CommentMultiline
- CommentSingle
- CommentSpecial
- )
-
- // Preprocessor "comments".
- const (
- CommentPreproc TokenType = 6100 + iota
- CommentPreprocFile
- )
-
- // Generic tokens.
- const (
- Generic TokenType = 7000 + iota
- GenericDeleted
- GenericEmph
- GenericError
- GenericHeading
- GenericInserted
- GenericOutput
- GenericPrompt
- GenericStrong
- GenericSubheading
- GenericTraceback
- GenericUnderline
- )
-
- // Text.
- const (
- Text TokenType = 8000 + iota
- TextWhitespace
- TextSymbol
- TextPunctuation
- )
-
- // Aliases.
- const (
- Whitespace = TextWhitespace
-
- Date = LiteralDate
-
- String = LiteralString
- StringAffix = LiteralStringAffix
- StringBacktick = LiteralStringBacktick
- StringChar = LiteralStringChar
- StringDelimiter = LiteralStringDelimiter
- StringDoc = LiteralStringDoc
- StringDouble = LiteralStringDouble
- StringEscape = LiteralStringEscape
- StringHeredoc = LiteralStringHeredoc
- StringInterpol = LiteralStringInterpol
- StringOther = LiteralStringOther
- StringRegex = LiteralStringRegex
- StringSingle = LiteralStringSingle
- StringSymbol = LiteralStringSymbol
-
- Number = LiteralNumber
- NumberBin = LiteralNumberBin
- NumberFloat = LiteralNumberFloat
- NumberHex = LiteralNumberHex
- NumberInteger = LiteralNumberInteger
- NumberIntegerLong = LiteralNumberIntegerLong
- NumberOct = LiteralNumberOct
- )
-
- var (
- StandardTypes = map[TokenType]string{
- Background: "chroma",
- LineNumbers: "ln",
- LineNumbersTable: "lnt",
- LineHighlight: "hl",
- LineTable: "lntable",
- LineTableTD: "lntd",
- Text: "",
- Whitespace: "w",
- Error: "err",
- Other: "x",
- // I have no idea what this is used for...
- // Escape: "esc",
-
- Keyword: "k",
- KeywordConstant: "kc",
- KeywordDeclaration: "kd",
- KeywordNamespace: "kn",
- KeywordPseudo: "kp",
- KeywordReserved: "kr",
- KeywordType: "kt",
-
- Name: "n",
- NameAttribute: "na",
- NameBuiltin: "nb",
- NameBuiltinPseudo: "bp",
- NameClass: "nc",
- NameConstant: "no",
- NameDecorator: "nd",
- NameEntity: "ni",
- NameException: "ne",
- NameFunction: "nf",
- NameFunctionMagic: "fm",
- NameProperty: "py",
- NameLabel: "nl",
- NameNamespace: "nn",
- NameOther: "nx",
- NameTag: "nt",
- NameVariable: "nv",
- NameVariableClass: "vc",
- NameVariableGlobal: "vg",
- NameVariableInstance: "vi",
- NameVariableMagic: "vm",
-
- Literal: "l",
- LiteralDate: "ld",
-
- String: "s",
- StringAffix: "sa",
- StringBacktick: "sb",
- StringChar: "sc",
- StringDelimiter: "dl",
- StringDoc: "sd",
- StringDouble: "s2",
- StringEscape: "se",
- StringHeredoc: "sh",
- StringInterpol: "si",
- StringOther: "sx",
- StringRegex: "sr",
- StringSingle: "s1",
- StringSymbol: "ss",
-
- Number: "m",
- NumberBin: "mb",
- NumberFloat: "mf",
- NumberHex: "mh",
- NumberInteger: "mi",
- NumberIntegerLong: "il",
- NumberOct: "mo",
-
- Operator: "o",
- OperatorWord: "ow",
-
- Punctuation: "p",
-
- Comment: "c",
- CommentHashbang: "ch",
- CommentMultiline: "cm",
- CommentPreproc: "cp",
- CommentPreprocFile: "cpf",
- CommentSingle: "c1",
- CommentSpecial: "cs",
-
- Generic: "g",
- GenericDeleted: "gd",
- GenericEmph: "ge",
- GenericError: "gr",
- GenericHeading: "gh",
- GenericInserted: "gi",
- GenericOutput: "go",
- GenericPrompt: "gp",
- GenericStrong: "gs",
- GenericSubheading: "gu",
- GenericTraceback: "gt",
- GenericUnderline: "gl",
- }
- )
-
- func (t TokenType) Parent() TokenType {
- if t%100 != 0 {
- return t / 100 * 100
- }
- if t%1000 != 0 {
- return t / 1000 * 1000
- }
- return 0
- }
-
- func (t TokenType) Category() TokenType {
- return t / 1000 * 1000
- }
-
- func (t TokenType) SubCategory() TokenType {
- return t / 100 * 100
- }
-
- func (t TokenType) InCategory(other TokenType) bool {
- return t/1000 == other/1000
- }
-
- func (t TokenType) InSubCategory(other TokenType) bool {
- return t/100 == other/100
- }
-
- func (t TokenType) Emit(groups []string, lexer Lexer) Iterator {
- return Literator(Token{Type: t, Value: groups[0]})
- }
|