You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

token.go 2.0KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144
  1. package toml
  2. import (
  3. "fmt"
  4. "strconv"
  5. "unicode"
  6. )
  7. // Define tokens
  8. type tokenType int
  9. const (
  10. eof = -(iota + 1)
  11. )
  12. const (
  13. tokenError tokenType = iota
  14. tokenEOF
  15. tokenComment
  16. tokenKey
  17. tokenString
  18. tokenInteger
  19. tokenTrue
  20. tokenFalse
  21. tokenFloat
  22. tokenInf
  23. tokenNan
  24. tokenEqual
  25. tokenLeftBracket
  26. tokenRightBracket
  27. tokenLeftCurlyBrace
  28. tokenRightCurlyBrace
  29. tokenLeftParen
  30. tokenRightParen
  31. tokenDoubleLeftBracket
  32. tokenDoubleRightBracket
  33. tokenDate
  34. tokenKeyGroup
  35. tokenKeyGroupArray
  36. tokenComma
  37. tokenColon
  38. tokenDollar
  39. tokenStar
  40. tokenQuestion
  41. tokenDot
  42. tokenDotDot
  43. tokenEOL
  44. )
  45. var tokenTypeNames = []string{
  46. "Error",
  47. "EOF",
  48. "Comment",
  49. "Key",
  50. "String",
  51. "Integer",
  52. "True",
  53. "False",
  54. "Float",
  55. "Inf",
  56. "NaN",
  57. "=",
  58. "[",
  59. "]",
  60. "{",
  61. "}",
  62. "(",
  63. ")",
  64. "]]",
  65. "[[",
  66. "Date",
  67. "KeyGroup",
  68. "KeyGroupArray",
  69. ",",
  70. ":",
  71. "$",
  72. "*",
  73. "?",
  74. ".",
  75. "..",
  76. "EOL",
  77. }
  78. type token struct {
  79. Position
  80. typ tokenType
  81. val string
  82. }
  83. func (tt tokenType) String() string {
  84. idx := int(tt)
  85. if idx < len(tokenTypeNames) {
  86. return tokenTypeNames[idx]
  87. }
  88. return "Unknown"
  89. }
  90. func (t token) Int() int {
  91. if result, err := strconv.Atoi(t.val); err != nil {
  92. panic(err)
  93. } else {
  94. return result
  95. }
  96. }
  97. func (t token) String() string {
  98. switch t.typ {
  99. case tokenEOF:
  100. return "EOF"
  101. case tokenError:
  102. return t.val
  103. }
  104. return fmt.Sprintf("%q", t.val)
  105. }
  106. func isSpace(r rune) bool {
  107. return r == ' ' || r == '\t'
  108. }
  109. func isAlphanumeric(r rune) bool {
  110. return unicode.IsLetter(r) || r == '_'
  111. }
  112. func isKeyChar(r rune) bool {
  113. // Keys start with the first character that isn't whitespace or [ and end
  114. // with the last non-whitespace character before the equals sign. Keys
  115. // cannot contain a # character."
  116. return !(r == '\r' || r == '\n' || r == eof || r == '=')
  117. }
  118. func isKeyStartChar(r rune) bool {
  119. return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '[')
  120. }
  121. func isDigit(r rune) bool {
  122. return unicode.IsNumber(r)
  123. }
  124. func isHexDigit(r rune) bool {
  125. return isDigit(r) ||
  126. (r >= 'a' && r <= 'f') ||
  127. (r >= 'A' && r <= 'F')
  128. }