You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

encoder.go 4.9KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225
  1. package packfile
  2. import (
  3. "compress/zlib"
  4. "crypto/sha1"
  5. "fmt"
  6. "io"
  7. "github.com/go-git/go-git/v5/plumbing"
  8. "github.com/go-git/go-git/v5/plumbing/storer"
  9. "github.com/go-git/go-git/v5/utils/binary"
  10. "github.com/go-git/go-git/v5/utils/ioutil"
  11. )
  12. // Encoder gets the data from the storage and write it into the writer in PACK
  13. // format
  14. type Encoder struct {
  15. selector *deltaSelector
  16. w *offsetWriter
  17. zw *zlib.Writer
  18. hasher plumbing.Hasher
  19. useRefDeltas bool
  20. }
  21. // NewEncoder creates a new packfile encoder using a specific Writer and
  22. // EncodedObjectStorer. By default deltas used to generate the packfile will be
  23. // OFSDeltaObject. To use Reference deltas, set useRefDeltas to true.
  24. func NewEncoder(w io.Writer, s storer.EncodedObjectStorer, useRefDeltas bool) *Encoder {
  25. h := plumbing.Hasher{
  26. Hash: sha1.New(),
  27. }
  28. mw := io.MultiWriter(w, h)
  29. ow := newOffsetWriter(mw)
  30. zw := zlib.NewWriter(mw)
  31. return &Encoder{
  32. selector: newDeltaSelector(s),
  33. w: ow,
  34. zw: zw,
  35. hasher: h,
  36. useRefDeltas: useRefDeltas,
  37. }
  38. }
  39. // Encode creates a packfile containing all the objects referenced in
  40. // hashes and writes it to the writer in the Encoder. `packWindow`
  41. // specifies the size of the sliding window used to compare objects
  42. // for delta compression; 0 turns off delta compression entirely.
  43. func (e *Encoder) Encode(
  44. hashes []plumbing.Hash,
  45. packWindow uint,
  46. ) (plumbing.Hash, error) {
  47. objects, err := e.selector.ObjectsToPack(hashes, packWindow)
  48. if err != nil {
  49. return plumbing.ZeroHash, err
  50. }
  51. return e.encode(objects)
  52. }
  53. func (e *Encoder) encode(objects []*ObjectToPack) (plumbing.Hash, error) {
  54. if err := e.head(len(objects)); err != nil {
  55. return plumbing.ZeroHash, err
  56. }
  57. for _, o := range objects {
  58. if err := e.entry(o); err != nil {
  59. return plumbing.ZeroHash, err
  60. }
  61. }
  62. return e.footer()
  63. }
  64. func (e *Encoder) head(numEntries int) error {
  65. return binary.Write(
  66. e.w,
  67. signature,
  68. int32(VersionSupported),
  69. int32(numEntries),
  70. )
  71. }
  72. func (e *Encoder) entry(o *ObjectToPack) (err error) {
  73. if o.WantWrite() {
  74. // A cycle exists in this delta chain. This should only occur if a
  75. // selected object representation disappeared during writing
  76. // (for example due to a concurrent repack) and a different base
  77. // was chosen, forcing a cycle. Select something other than a
  78. // delta, and write this object.
  79. e.selector.restoreOriginal(o)
  80. o.BackToOriginal()
  81. }
  82. if o.IsWritten() {
  83. return nil
  84. }
  85. o.MarkWantWrite()
  86. if err := e.writeBaseIfDelta(o); err != nil {
  87. return err
  88. }
  89. // We need to check if we already write that object due a cyclic delta chain
  90. if o.IsWritten() {
  91. return nil
  92. }
  93. o.Offset = e.w.Offset()
  94. if o.IsDelta() {
  95. if err := e.writeDeltaHeader(o); err != nil {
  96. return err
  97. }
  98. } else {
  99. if err := e.entryHead(o.Type(), o.Size()); err != nil {
  100. return err
  101. }
  102. }
  103. e.zw.Reset(e.w)
  104. defer ioutil.CheckClose(e.zw, &err)
  105. or, err := o.Object.Reader()
  106. if err != nil {
  107. return err
  108. }
  109. defer ioutil.CheckClose(or, &err)
  110. _, err = io.Copy(e.zw, or)
  111. if err != nil {
  112. return err
  113. }
  114. return nil
  115. }
  116. func (e *Encoder) writeBaseIfDelta(o *ObjectToPack) error {
  117. if o.IsDelta() && !o.Base.IsWritten() {
  118. // We must write base first
  119. return e.entry(o.Base)
  120. }
  121. return nil
  122. }
  123. func (e *Encoder) writeDeltaHeader(o *ObjectToPack) error {
  124. // Write offset deltas by default
  125. t := plumbing.OFSDeltaObject
  126. if e.useRefDeltas {
  127. t = plumbing.REFDeltaObject
  128. }
  129. if err := e.entryHead(t, o.Object.Size()); err != nil {
  130. return err
  131. }
  132. if e.useRefDeltas {
  133. return e.writeRefDeltaHeader(o.Base.Hash())
  134. } else {
  135. return e.writeOfsDeltaHeader(o)
  136. }
  137. }
  138. func (e *Encoder) writeRefDeltaHeader(base plumbing.Hash) error {
  139. return binary.Write(e.w, base)
  140. }
  141. func (e *Encoder) writeOfsDeltaHeader(o *ObjectToPack) error {
  142. // for OFS_DELTA, offset of the base is interpreted as negative offset
  143. // relative to the type-byte of the header of the ofs-delta entry.
  144. relativeOffset := o.Offset - o.Base.Offset
  145. if relativeOffset <= 0 {
  146. return fmt.Errorf("bad offset for OFS_DELTA entry: %d", relativeOffset)
  147. }
  148. return binary.WriteVariableWidthInt(e.w, relativeOffset)
  149. }
  150. func (e *Encoder) entryHead(typeNum plumbing.ObjectType, size int64) error {
  151. t := int64(typeNum)
  152. header := []byte{}
  153. c := (t << firstLengthBits) | (size & maskFirstLength)
  154. size >>= firstLengthBits
  155. for {
  156. if size == 0 {
  157. break
  158. }
  159. header = append(header, byte(c|maskContinue))
  160. c = size & int64(maskLength)
  161. size >>= lengthBits
  162. }
  163. header = append(header, byte(c))
  164. _, err := e.w.Write(header)
  165. return err
  166. }
  167. func (e *Encoder) footer() (plumbing.Hash, error) {
  168. h := e.hasher.Sum()
  169. return h, binary.Write(e.w, h)
  170. }
  171. type offsetWriter struct {
  172. w io.Writer
  173. offset int64
  174. }
  175. func newOffsetWriter(w io.Writer) *offsetWriter {
  176. return &offsetWriter{w: w}
  177. }
  178. func (ow *offsetWriter) Write(p []byte) (n int, err error) {
  179. n, err = ow.w.Write(p)
  180. ow.offset += int64(n)
  181. return n, err
  182. }
  183. func (ow *offsetWriter) Offset() int64 {
  184. return ow.offset
  185. }