You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

write.go 3.6KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145
  1. // Copyright (c) 2017 Couchbase, Inc.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package zap
  15. import (
  16. "encoding/binary"
  17. "io"
  18. "github.com/RoaringBitmap/roaring"
  19. )
  20. // writes out the length of the roaring bitmap in bytes as varint
  21. // then writes out the roaring bitmap itself
  22. func writeRoaringWithLen(r *roaring.Bitmap, w io.Writer,
  23. reuseBufVarint []byte) (int, error) {
  24. buf, err := r.ToBytes()
  25. if err != nil {
  26. return 0, err
  27. }
  28. var tw int
  29. // write out the length
  30. n := binary.PutUvarint(reuseBufVarint, uint64(len(buf)))
  31. nw, err := w.Write(reuseBufVarint[:n])
  32. tw += nw
  33. if err != nil {
  34. return tw, err
  35. }
  36. // write out the roaring bytes
  37. nw, err = w.Write(buf)
  38. tw += nw
  39. if err != nil {
  40. return tw, err
  41. }
  42. return tw, nil
  43. }
  44. func persistFields(fieldsInv []string, w *CountHashWriter, dictLocs []uint64) (uint64, error) {
  45. var rv uint64
  46. var fieldsOffsets []uint64
  47. for fieldID, fieldName := range fieldsInv {
  48. // record start of this field
  49. fieldsOffsets = append(fieldsOffsets, uint64(w.Count()))
  50. // write out the dict location and field name length
  51. _, err := writeUvarints(w, dictLocs[fieldID], uint64(len(fieldName)))
  52. if err != nil {
  53. return 0, err
  54. }
  55. // write out the field name
  56. _, err = w.Write([]byte(fieldName))
  57. if err != nil {
  58. return 0, err
  59. }
  60. }
  61. // now write out the fields index
  62. rv = uint64(w.Count())
  63. for fieldID := range fieldsInv {
  64. err := binary.Write(w, binary.BigEndian, fieldsOffsets[fieldID])
  65. if err != nil {
  66. return 0, err
  67. }
  68. }
  69. return rv, nil
  70. }
  71. // FooterSize is the size of the footer record in bytes
  72. // crc + ver + chunk + field offset + stored offset + num docs + docValueOffset
  73. const FooterSize = 4 + 4 + 4 + 8 + 8 + 8 + 8
  74. func persistFooter(numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset uint64,
  75. chunkFactor uint32, crcBeforeFooter uint32, writerIn io.Writer) error {
  76. w := NewCountHashWriter(writerIn)
  77. w.crc = crcBeforeFooter
  78. // write out the number of docs
  79. err := binary.Write(w, binary.BigEndian, numDocs)
  80. if err != nil {
  81. return err
  82. }
  83. // write out the stored field index location:
  84. err = binary.Write(w, binary.BigEndian, storedIndexOffset)
  85. if err != nil {
  86. return err
  87. }
  88. // write out the field index location
  89. err = binary.Write(w, binary.BigEndian, fieldsIndexOffset)
  90. if err != nil {
  91. return err
  92. }
  93. // write out the fieldDocValue location
  94. err = binary.Write(w, binary.BigEndian, docValueOffset)
  95. if err != nil {
  96. return err
  97. }
  98. // write out 32-bit chunk factor
  99. err = binary.Write(w, binary.BigEndian, chunkFactor)
  100. if err != nil {
  101. return err
  102. }
  103. // write out 32-bit version
  104. err = binary.Write(w, binary.BigEndian, Version)
  105. if err != nil {
  106. return err
  107. }
  108. // write out CRC-32 of everything upto but not including this CRC
  109. err = binary.Write(w, binary.BigEndian, w.crc)
  110. if err != nil {
  111. return err
  112. }
  113. return nil
  114. }
  115. func writeUvarints(w io.Writer, vals ...uint64) (tw int, err error) {
  116. buf := make([]byte, binary.MaxVarintLen64)
  117. for _, val := range vals {
  118. n := binary.PutUvarint(buf, val)
  119. var nw int
  120. nw, err = w.Write(buf[:n])
  121. tw += nw
  122. if err != nil {
  123. return tw, err
  124. }
  125. }
  126. return tw, err
  127. }