tokenize.go 2.2 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192
  1. package ini
  2. import (
  3. "strings"
  4. )
  5. func tokenize(lines []string) ([]lineToken, error) {
  6. tokens := make([]lineToken, 0, len(lines))
  7. for _, line := range lines {
  8. if len(strings.TrimSpace(line)) == 0 || isLineComment(line) {
  9. continue
  10. }
  11. if tok := asProfile(line); tok != nil {
  12. tokens = append(tokens, tok)
  13. } else if tok := asProperty(line); tok != nil {
  14. tokens = append(tokens, tok)
  15. } else if tok := asSubProperty(line); tok != nil {
  16. tokens = append(tokens, tok)
  17. } else if tok := asContinuation(line); tok != nil {
  18. tokens = append(tokens, tok)
  19. } // unrecognized tokens are effectively ignored
  20. }
  21. return tokens, nil
  22. }
  23. func isLineComment(line string) bool {
  24. trimmed := strings.TrimLeft(line, " \t")
  25. return strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, ";")
  26. }
  27. func asProfile(line string) *lineTokenProfile { // " [ type name ] ; comment"
  28. trimmed := strings.TrimSpace(trimProfileComment(line)) // "[ type name ]"
  29. if !isBracketed(trimmed) {
  30. return nil
  31. }
  32. trimmed = trimmed[1 : len(trimmed)-1] // " type name " (or just " name ")
  33. trimmed = strings.TrimSpace(trimmed) // "type name" / "name"
  34. typ, name := splitProfile(trimmed)
  35. return &lineTokenProfile{
  36. Type: typ,
  37. Name: name,
  38. }
  39. }
  40. func asProperty(line string) *lineTokenProperty {
  41. if isLineSpace(rune(line[0])) {
  42. return nil
  43. }
  44. trimmed := trimPropertyComment(line)
  45. trimmed = strings.TrimRight(trimmed, " \t")
  46. k, v, ok := splitProperty(trimmed)
  47. if !ok {
  48. return nil
  49. }
  50. return &lineTokenProperty{
  51. Key: strings.ToLower(k), // LEGACY: normalize key case
  52. Value: legacyStrconv(v), // LEGACY: see func docs
  53. }
  54. }
  55. func asSubProperty(line string) *lineTokenSubProperty {
  56. if !isLineSpace(rune(line[0])) {
  57. return nil
  58. }
  59. // comments on sub-properties are included in the value
  60. trimmed := strings.TrimLeft(line, " \t")
  61. k, v, ok := splitProperty(trimmed)
  62. if !ok {
  63. return nil
  64. }
  65. return &lineTokenSubProperty{ // same LEGACY constraints as in normal property
  66. Key: strings.ToLower(k),
  67. Value: legacyStrconv(v),
  68. }
  69. }
  70. func asContinuation(line string) *lineTokenContinuation {
  71. if !isLineSpace(rune(line[0])) {
  72. return nil
  73. }
  74. // includes comments like sub-properties
  75. trimmed := strings.TrimLeft(line, " \t")
  76. return &lineTokenContinuation{
  77. Value: trimmed,
  78. }
  79. }