You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

token.go 1.9KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136
  1. package toml
  2. import "fmt"
  3. // Define tokens
  4. type tokenType int
  5. const (
  6. eof = -(iota + 1)
  7. )
  8. const (
  9. tokenError tokenType = iota
  10. tokenEOF
  11. tokenComment
  12. tokenKey
  13. tokenString
  14. tokenInteger
  15. tokenTrue
  16. tokenFalse
  17. tokenFloat
  18. tokenInf
  19. tokenNan
  20. tokenEqual
  21. tokenLeftBracket
  22. tokenRightBracket
  23. tokenLeftCurlyBrace
  24. tokenRightCurlyBrace
  25. tokenLeftParen
  26. tokenRightParen
  27. tokenDoubleLeftBracket
  28. tokenDoubleRightBracket
  29. tokenLocalDate
  30. tokenLocalTime
  31. tokenTimeOffset
  32. tokenKeyGroup
  33. tokenKeyGroupArray
  34. tokenComma
  35. tokenColon
  36. tokenDollar
  37. tokenStar
  38. tokenQuestion
  39. tokenDot
  40. tokenDotDot
  41. tokenEOL
  42. )
  43. var tokenTypeNames = []string{
  44. "Error",
  45. "EOF",
  46. "Comment",
  47. "Key",
  48. "String",
  49. "Integer",
  50. "True",
  51. "False",
  52. "Float",
  53. "Inf",
  54. "NaN",
  55. "=",
  56. "[",
  57. "]",
  58. "{",
  59. "}",
  60. "(",
  61. ")",
  62. "]]",
  63. "[[",
  64. "LocalDate",
  65. "LocalTime",
  66. "TimeOffset",
  67. "KeyGroup",
  68. "KeyGroupArray",
  69. ",",
  70. ":",
  71. "$",
  72. "*",
  73. "?",
  74. ".",
  75. "..",
  76. "EOL",
  77. }
  78. type token struct {
  79. Position
  80. typ tokenType
  81. val string
  82. }
  83. func (tt tokenType) String() string {
  84. idx := int(tt)
  85. if idx < len(tokenTypeNames) {
  86. return tokenTypeNames[idx]
  87. }
  88. return "Unknown"
  89. }
  90. func (t token) String() string {
  91. switch t.typ {
  92. case tokenEOF:
  93. return "EOF"
  94. case tokenError:
  95. return t.val
  96. }
  97. return fmt.Sprintf("%q", t.val)
  98. }
  99. func isSpace(r rune) bool {
  100. return r == ' ' || r == '\t'
  101. }
  102. func isAlphanumeric(r rune) bool {
  103. return 'a' <= r && r <= 'z' || 'A' <= r && r <= 'Z' || r == '_'
  104. }
  105. func isKeyChar(r rune) bool {
  106. // Keys start with the first character that isn't whitespace or [ and end
  107. // with the last non-whitespace character before the equals sign. Keys
  108. // cannot contain a # character."
  109. return !(r == '\r' || r == '\n' || r == eof || r == '=')
  110. }
  111. func isKeyStartChar(r rune) bool {
  112. return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '[')
  113. }
  114. func isDigit(r rune) bool {
  115. return '0' <= r && r <= '9'
  116. }
  117. func isHexDigit(r rune) bool {
  118. return isDigit(r) ||
  119. (r >= 'a' && r <= 'f') ||
  120. (r >= 'A' && r <= 'F')
  121. }