You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

encode.go 14KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577
  1. //
  2. // Copyright (c) 2011-2019 Canonical Ltd
  3. //
  4. // Licensed under the Apache License, Version 2.0 (the "License");
  5. // you may not use this file except in compliance with the License.
  6. // You may obtain a copy of the License at
  7. //
  8. // http://www.apache.org/licenses/LICENSE-2.0
  9. //
  10. // Unless required by applicable law or agreed to in writing, software
  11. // distributed under the License is distributed on an "AS IS" BASIS,
  12. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. // See the License for the specific language governing permissions and
  14. // limitations under the License.
  15. package yaml
  16. import (
  17. "encoding"
  18. "fmt"
  19. "io"
  20. "reflect"
  21. "regexp"
  22. "sort"
  23. "strconv"
  24. "strings"
  25. "time"
  26. "unicode/utf8"
  27. )
  28. type encoder struct {
  29. emitter yaml_emitter_t
  30. event yaml_event_t
  31. out []byte
  32. flow bool
  33. indent int
  34. doneInit bool
  35. }
  36. func newEncoder() *encoder {
  37. e := &encoder{}
  38. yaml_emitter_initialize(&e.emitter)
  39. yaml_emitter_set_output_string(&e.emitter, &e.out)
  40. yaml_emitter_set_unicode(&e.emitter, true)
  41. return e
  42. }
  43. func newEncoderWithWriter(w io.Writer) *encoder {
  44. e := &encoder{}
  45. yaml_emitter_initialize(&e.emitter)
  46. yaml_emitter_set_output_writer(&e.emitter, w)
  47. yaml_emitter_set_unicode(&e.emitter, true)
  48. return e
  49. }
  50. func (e *encoder) init() {
  51. if e.doneInit {
  52. return
  53. }
  54. if e.indent == 0 {
  55. e.indent = 4
  56. }
  57. e.emitter.best_indent = e.indent
  58. yaml_stream_start_event_initialize(&e.event, yaml_UTF8_ENCODING)
  59. e.emit()
  60. e.doneInit = true
  61. }
  62. func (e *encoder) finish() {
  63. e.emitter.open_ended = false
  64. yaml_stream_end_event_initialize(&e.event)
  65. e.emit()
  66. }
  67. func (e *encoder) destroy() {
  68. yaml_emitter_delete(&e.emitter)
  69. }
  70. func (e *encoder) emit() {
  71. // This will internally delete the e.event value.
  72. e.must(yaml_emitter_emit(&e.emitter, &e.event))
  73. }
  74. func (e *encoder) must(ok bool) {
  75. if !ok {
  76. msg := e.emitter.problem
  77. if msg == "" {
  78. msg = "unknown problem generating YAML content"
  79. }
  80. failf("%s", msg)
  81. }
  82. }
  83. func (e *encoder) marshalDoc(tag string, in reflect.Value) {
  84. e.init()
  85. var node *Node
  86. if in.IsValid() {
  87. node, _ = in.Interface().(*Node)
  88. }
  89. if node != nil && node.Kind == DocumentNode {
  90. e.nodev(in)
  91. } else {
  92. yaml_document_start_event_initialize(&e.event, nil, nil, true)
  93. e.emit()
  94. e.marshal(tag, in)
  95. yaml_document_end_event_initialize(&e.event, true)
  96. e.emit()
  97. }
  98. }
  99. func (e *encoder) marshal(tag string, in reflect.Value) {
  100. tag = shortTag(tag)
  101. if !in.IsValid() || in.Kind() == reflect.Ptr && in.IsNil() {
  102. e.nilv()
  103. return
  104. }
  105. iface := in.Interface()
  106. switch value := iface.(type) {
  107. case *Node:
  108. e.nodev(in)
  109. return
  110. case Node:
  111. if !in.CanAddr() {
  112. var n = reflect.New(in.Type()).Elem()
  113. n.Set(in)
  114. in = n
  115. }
  116. e.nodev(in.Addr())
  117. return
  118. case time.Time:
  119. e.timev(tag, in)
  120. return
  121. case *time.Time:
  122. e.timev(tag, in.Elem())
  123. return
  124. case time.Duration:
  125. e.stringv(tag, reflect.ValueOf(value.String()))
  126. return
  127. case Marshaler:
  128. v, err := value.MarshalYAML()
  129. if err != nil {
  130. fail(err)
  131. }
  132. if v == nil {
  133. e.nilv()
  134. return
  135. }
  136. e.marshal(tag, reflect.ValueOf(v))
  137. return
  138. case encoding.TextMarshaler:
  139. text, err := value.MarshalText()
  140. if err != nil {
  141. fail(err)
  142. }
  143. in = reflect.ValueOf(string(text))
  144. case nil:
  145. e.nilv()
  146. return
  147. }
  148. switch in.Kind() {
  149. case reflect.Interface:
  150. e.marshal(tag, in.Elem())
  151. case reflect.Map:
  152. e.mapv(tag, in)
  153. case reflect.Ptr:
  154. e.marshal(tag, in.Elem())
  155. case reflect.Struct:
  156. e.structv(tag, in)
  157. case reflect.Slice, reflect.Array:
  158. e.slicev(tag, in)
  159. case reflect.String:
  160. e.stringv(tag, in)
  161. case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
  162. e.intv(tag, in)
  163. case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
  164. e.uintv(tag, in)
  165. case reflect.Float32, reflect.Float64:
  166. e.floatv(tag, in)
  167. case reflect.Bool:
  168. e.boolv(tag, in)
  169. default:
  170. panic("cannot marshal type: " + in.Type().String())
  171. }
  172. }
  173. func (e *encoder) mapv(tag string, in reflect.Value) {
  174. e.mappingv(tag, func() {
  175. keys := keyList(in.MapKeys())
  176. sort.Sort(keys)
  177. for _, k := range keys {
  178. e.marshal("", k)
  179. e.marshal("", in.MapIndex(k))
  180. }
  181. })
  182. }
  183. func (e *encoder) fieldByIndex(v reflect.Value, index []int) (field reflect.Value) {
  184. for _, num := range index {
  185. for {
  186. if v.Kind() == reflect.Ptr {
  187. if v.IsNil() {
  188. return reflect.Value{}
  189. }
  190. v = v.Elem()
  191. continue
  192. }
  193. break
  194. }
  195. v = v.Field(num)
  196. }
  197. return v
  198. }
  199. func (e *encoder) structv(tag string, in reflect.Value) {
  200. sinfo, err := getStructInfo(in.Type())
  201. if err != nil {
  202. panic(err)
  203. }
  204. e.mappingv(tag, func() {
  205. for _, info := range sinfo.FieldsList {
  206. var value reflect.Value
  207. if info.Inline == nil {
  208. value = in.Field(info.Num)
  209. } else {
  210. value = e.fieldByIndex(in, info.Inline)
  211. if !value.IsValid() {
  212. continue
  213. }
  214. }
  215. if info.OmitEmpty && isZero(value) {
  216. continue
  217. }
  218. e.marshal("", reflect.ValueOf(info.Key))
  219. e.flow = info.Flow
  220. e.marshal("", value)
  221. }
  222. if sinfo.InlineMap >= 0 {
  223. m := in.Field(sinfo.InlineMap)
  224. if m.Len() > 0 {
  225. e.flow = false
  226. keys := keyList(m.MapKeys())
  227. sort.Sort(keys)
  228. for _, k := range keys {
  229. if _, found := sinfo.FieldsMap[k.String()]; found {
  230. panic(fmt.Sprintf("cannot have key %q in inlined map: conflicts with struct field", k.String()))
  231. }
  232. e.marshal("", k)
  233. e.flow = false
  234. e.marshal("", m.MapIndex(k))
  235. }
  236. }
  237. }
  238. })
  239. }
  240. func (e *encoder) mappingv(tag string, f func()) {
  241. implicit := tag == ""
  242. style := yaml_BLOCK_MAPPING_STYLE
  243. if e.flow {
  244. e.flow = false
  245. style = yaml_FLOW_MAPPING_STYLE
  246. }
  247. yaml_mapping_start_event_initialize(&e.event, nil, []byte(tag), implicit, style)
  248. e.emit()
  249. f()
  250. yaml_mapping_end_event_initialize(&e.event)
  251. e.emit()
  252. }
  253. func (e *encoder) slicev(tag string, in reflect.Value) {
  254. implicit := tag == ""
  255. style := yaml_BLOCK_SEQUENCE_STYLE
  256. if e.flow {
  257. e.flow = false
  258. style = yaml_FLOW_SEQUENCE_STYLE
  259. }
  260. e.must(yaml_sequence_start_event_initialize(&e.event, nil, []byte(tag), implicit, style))
  261. e.emit()
  262. n := in.Len()
  263. for i := 0; i < n; i++ {
  264. e.marshal("", in.Index(i))
  265. }
  266. e.must(yaml_sequence_end_event_initialize(&e.event))
  267. e.emit()
  268. }
  269. // isBase60 returns whether s is in base 60 notation as defined in YAML 1.1.
  270. //
  271. // The base 60 float notation in YAML 1.1 is a terrible idea and is unsupported
  272. // in YAML 1.2 and by this package, but these should be marshalled quoted for
  273. // the time being for compatibility with other parsers.
  274. func isBase60Float(s string) (result bool) {
  275. // Fast path.
  276. if s == "" {
  277. return false
  278. }
  279. c := s[0]
  280. if !(c == '+' || c == '-' || c >= '0' && c <= '9') || strings.IndexByte(s, ':') < 0 {
  281. return false
  282. }
  283. // Do the full match.
  284. return base60float.MatchString(s)
  285. }
  286. // From http://yaml.org/type/float.html, except the regular expression there
  287. // is bogus. In practice parsers do not enforce the "\.[0-9_]*" suffix.
  288. var base60float = regexp.MustCompile(`^[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+(?:\.[0-9_]*)?$`)
  289. // isOldBool returns whether s is bool notation as defined in YAML 1.1.
  290. //
  291. // We continue to force strings that YAML 1.1 would interpret as booleans to be
  292. // rendered as quotes strings so that the marshalled output valid for YAML 1.1
  293. // parsing.
  294. func isOldBool(s string) (result bool) {
  295. switch s {
  296. case "y", "Y", "yes", "Yes", "YES", "on", "On", "ON",
  297. "n", "N", "no", "No", "NO", "off", "Off", "OFF":
  298. return true
  299. default:
  300. return false
  301. }
  302. }
  303. func (e *encoder) stringv(tag string, in reflect.Value) {
  304. var style yaml_scalar_style_t
  305. s := in.String()
  306. canUsePlain := true
  307. switch {
  308. case !utf8.ValidString(s):
  309. if tag == binaryTag {
  310. failf("explicitly tagged !!binary data must be base64-encoded")
  311. }
  312. if tag != "" {
  313. failf("cannot marshal invalid UTF-8 data as %s", shortTag(tag))
  314. }
  315. // It can't be encoded directly as YAML so use a binary tag
  316. // and encode it as base64.
  317. tag = binaryTag
  318. s = encodeBase64(s)
  319. case tag == "":
  320. // Check to see if it would resolve to a specific
  321. // tag when encoded unquoted. If it doesn't,
  322. // there's no need to quote it.
  323. rtag, _ := resolve("", s)
  324. canUsePlain = rtag == strTag && !(isBase60Float(s) || isOldBool(s))
  325. }
  326. // Note: it's possible for user code to emit invalid YAML
  327. // if they explicitly specify a tag and a string containing
  328. // text that's incompatible with that tag.
  329. switch {
  330. case strings.Contains(s, "\n"):
  331. if e.flow {
  332. style = yaml_DOUBLE_QUOTED_SCALAR_STYLE
  333. } else {
  334. style = yaml_LITERAL_SCALAR_STYLE
  335. }
  336. case canUsePlain:
  337. style = yaml_PLAIN_SCALAR_STYLE
  338. default:
  339. style = yaml_DOUBLE_QUOTED_SCALAR_STYLE
  340. }
  341. e.emitScalar(s, "", tag, style, nil, nil, nil, nil)
  342. }
  343. func (e *encoder) boolv(tag string, in reflect.Value) {
  344. var s string
  345. if in.Bool() {
  346. s = "true"
  347. } else {
  348. s = "false"
  349. }
  350. e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil)
  351. }
  352. func (e *encoder) intv(tag string, in reflect.Value) {
  353. s := strconv.FormatInt(in.Int(), 10)
  354. e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil)
  355. }
  356. func (e *encoder) uintv(tag string, in reflect.Value) {
  357. s := strconv.FormatUint(in.Uint(), 10)
  358. e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil)
  359. }
  360. func (e *encoder) timev(tag string, in reflect.Value) {
  361. t := in.Interface().(time.Time)
  362. s := t.Format(time.RFC3339Nano)
  363. e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil)
  364. }
  365. func (e *encoder) floatv(tag string, in reflect.Value) {
  366. // Issue #352: When formatting, use the precision of the underlying value
  367. precision := 64
  368. if in.Kind() == reflect.Float32 {
  369. precision = 32
  370. }
  371. s := strconv.FormatFloat(in.Float(), 'g', -1, precision)
  372. switch s {
  373. case "+Inf":
  374. s = ".inf"
  375. case "-Inf":
  376. s = "-.inf"
  377. case "NaN":
  378. s = ".nan"
  379. }
  380. e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil)
  381. }
  382. func (e *encoder) nilv() {
  383. e.emitScalar("null", "", "", yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil)
  384. }
  385. func (e *encoder) emitScalar(value, anchor, tag string, style yaml_scalar_style_t, head, line, foot, tail []byte) {
  386. // TODO Kill this function. Replace all initialize calls by their underlining Go literals.
  387. implicit := tag == ""
  388. if !implicit {
  389. tag = longTag(tag)
  390. }
  391. e.must(yaml_scalar_event_initialize(&e.event, []byte(anchor), []byte(tag), []byte(value), implicit, implicit, style))
  392. e.event.head_comment = head
  393. e.event.line_comment = line
  394. e.event.foot_comment = foot
  395. e.event.tail_comment = tail
  396. e.emit()
  397. }
  398. func (e *encoder) nodev(in reflect.Value) {
  399. e.node(in.Interface().(*Node), "")
  400. }
  401. func (e *encoder) node(node *Node, tail string) {
  402. // Zero nodes behave as nil.
  403. if node.Kind == 0 && node.IsZero() {
  404. e.nilv()
  405. return
  406. }
  407. // If the tag was not explicitly requested, and dropping it won't change the
  408. // implicit tag of the value, don't include it in the presentation.
  409. var tag = node.Tag
  410. var stag = shortTag(tag)
  411. var forceQuoting bool
  412. if tag != "" && node.Style&TaggedStyle == 0 {
  413. if node.Kind == ScalarNode {
  414. if stag == strTag && node.Style&(SingleQuotedStyle|DoubleQuotedStyle|LiteralStyle|FoldedStyle) != 0 {
  415. tag = ""
  416. } else {
  417. rtag, _ := resolve("", node.Value)
  418. if rtag == stag {
  419. tag = ""
  420. } else if stag == strTag {
  421. tag = ""
  422. forceQuoting = true
  423. }
  424. }
  425. } else {
  426. var rtag string
  427. switch node.Kind {
  428. case MappingNode:
  429. rtag = mapTag
  430. case SequenceNode:
  431. rtag = seqTag
  432. }
  433. if rtag == stag {
  434. tag = ""
  435. }
  436. }
  437. }
  438. switch node.Kind {
  439. case DocumentNode:
  440. yaml_document_start_event_initialize(&e.event, nil, nil, true)
  441. e.event.head_comment = []byte(node.HeadComment)
  442. e.emit()
  443. for _, node := range node.Content {
  444. e.node(node, "")
  445. }
  446. yaml_document_end_event_initialize(&e.event, true)
  447. e.event.foot_comment = []byte(node.FootComment)
  448. e.emit()
  449. case SequenceNode:
  450. style := yaml_BLOCK_SEQUENCE_STYLE
  451. if node.Style&FlowStyle != 0 {
  452. style = yaml_FLOW_SEQUENCE_STYLE
  453. }
  454. e.must(yaml_sequence_start_event_initialize(&e.event, []byte(node.Anchor), []byte(longTag(tag)), tag == "", style))
  455. e.event.head_comment = []byte(node.HeadComment)
  456. e.emit()
  457. for _, node := range node.Content {
  458. e.node(node, "")
  459. }
  460. e.must(yaml_sequence_end_event_initialize(&e.event))
  461. e.event.line_comment = []byte(node.LineComment)
  462. e.event.foot_comment = []byte(node.FootComment)
  463. e.emit()
  464. case MappingNode:
  465. style := yaml_BLOCK_MAPPING_STYLE
  466. if node.Style&FlowStyle != 0 {
  467. style = yaml_FLOW_MAPPING_STYLE
  468. }
  469. yaml_mapping_start_event_initialize(&e.event, []byte(node.Anchor), []byte(longTag(tag)), tag == "", style)
  470. e.event.tail_comment = []byte(tail)
  471. e.event.head_comment = []byte(node.HeadComment)
  472. e.emit()
  473. // The tail logic below moves the foot comment of prior keys to the following key,
  474. // since the value for each key may be a nested structure and the foot needs to be
  475. // processed only the entirety of the value is streamed. The last tail is processed
  476. // with the mapping end event.
  477. var tail string
  478. for i := 0; i+1 < len(node.Content); i += 2 {
  479. k := node.Content[i]
  480. foot := k.FootComment
  481. if foot != "" {
  482. kopy := *k
  483. kopy.FootComment = ""
  484. k = &kopy
  485. }
  486. e.node(k, tail)
  487. tail = foot
  488. v := node.Content[i+1]
  489. e.node(v, "")
  490. }
  491. yaml_mapping_end_event_initialize(&e.event)
  492. e.event.tail_comment = []byte(tail)
  493. e.event.line_comment = []byte(node.LineComment)
  494. e.event.foot_comment = []byte(node.FootComment)
  495. e.emit()
  496. case AliasNode:
  497. yaml_alias_event_initialize(&e.event, []byte(node.Value))
  498. e.event.head_comment = []byte(node.HeadComment)
  499. e.event.line_comment = []byte(node.LineComment)
  500. e.event.foot_comment = []byte(node.FootComment)
  501. e.emit()
  502. case ScalarNode:
  503. value := node.Value
  504. if !utf8.ValidString(value) {
  505. if stag == binaryTag {
  506. failf("explicitly tagged !!binary data must be base64-encoded")
  507. }
  508. if stag != "" {
  509. failf("cannot marshal invalid UTF-8 data as %s", stag)
  510. }
  511. // It can't be encoded directly as YAML so use a binary tag
  512. // and encode it as base64.
  513. tag = binaryTag
  514. value = encodeBase64(value)
  515. }
  516. style := yaml_PLAIN_SCALAR_STYLE
  517. switch {
  518. case node.Style&DoubleQuotedStyle != 0:
  519. style = yaml_DOUBLE_QUOTED_SCALAR_STYLE
  520. case node.Style&SingleQuotedStyle != 0:
  521. style = yaml_SINGLE_QUOTED_SCALAR_STYLE
  522. case node.Style&LiteralStyle != 0:
  523. style = yaml_LITERAL_SCALAR_STYLE
  524. case node.Style&FoldedStyle != 0:
  525. style = yaml_FOLDED_SCALAR_STYLE
  526. case strings.Contains(value, "\n"):
  527. style = yaml_LITERAL_SCALAR_STYLE
  528. case forceQuoting:
  529. style = yaml_DOUBLE_QUOTED_SCALAR_STYLE
  530. }
  531. e.emitScalar(value, node.Anchor, tag, style, []byte(node.HeadComment), []byte(node.LineComment), []byte(node.FootComment), []byte(tail))
  532. default:
  533. failf("cannot encode node with unknown kind %d", node.Kind)
  534. }
  535. }