You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

snap.go 2.3KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100
  1. package unsnap
  2. import (
  3. "encoding/binary"
  4. // no c lib dependency
  5. snappy "github.com/golang/snappy"
  6. // or, use the C wrapper for speed
  7. //snappy "github.com/dgryski/go-csnappy"
  8. )
  9. // add Write() method for SnappyFile (see unsnap.go)
  10. // reference for snappy framing/streaming format:
  11. // http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
  12. // ?spec=svn68&r=71
  13. //
  14. // Write writes len(p) bytes from p to the underlying data stream.
  15. // It returns the number of bytes written from p (0 <= n <= len(p)) and
  16. // any error encountered that caused the write to stop early. Write
  17. // must return a non-nil error if it returns n < len(p).
  18. //
  19. func (sf *SnappyFile) Write(p []byte) (n int, err error) {
  20. if sf.SnappyEncodeDecodeOff {
  21. return sf.Writer.Write(p)
  22. }
  23. if !sf.Writing {
  24. panic("Writing on a read-only SnappyFile")
  25. }
  26. // encoding in snappy can apparently go beyond the original size, beware.
  27. // so our buffers must be sized 2*max snappy chunk => 2 * CHUNK_MAX(65536)
  28. sf.DecBuf.Reset()
  29. sf.EncBuf.Reset()
  30. if !sf.HeaderChunkWritten {
  31. sf.HeaderChunkWritten = true
  32. _, err = sf.Writer.Write(SnappyStreamHeaderMagic)
  33. if err != nil {
  34. return
  35. }
  36. }
  37. var chunk []byte
  38. var chunk_type byte
  39. var crc uint32
  40. for len(p) > 0 {
  41. // chunk points to input p by default, unencoded input.
  42. chunk = p[:IntMin(len(p), CHUNK_MAX)]
  43. crc = masked_crc32c(chunk)
  44. writeme := chunk[:]
  45. // first write to EncBuf, as a temp, in case we want
  46. // to discard and send uncompressed instead.
  47. compressed_chunk := snappy.Encode(sf.EncBuf.GetEndmostWritableSlice(), chunk)
  48. if len(compressed_chunk) <= int((1-_COMPRESSION_THRESHOLD)*float64(len(chunk))) {
  49. writeme = compressed_chunk
  50. chunk_type = _COMPRESSED_CHUNK
  51. } else {
  52. // keep writeme pointing at original chunk (uncompressed)
  53. chunk_type = _UNCOMPRESSED_CHUNK
  54. }
  55. const crc32Sz = 4
  56. var tag32 uint32 = uint32(chunk_type) + (uint32(len(writeme)+crc32Sz) << 8)
  57. err = binary.Write(sf.Writer, binary.LittleEndian, tag32)
  58. if err != nil {
  59. return
  60. }
  61. err = binary.Write(sf.Writer, binary.LittleEndian, crc)
  62. if err != nil {
  63. return
  64. }
  65. _, err = sf.Writer.Write(writeme)
  66. if err != nil {
  67. return
  68. }
  69. n += len(chunk)
  70. p = p[len(chunk):]
  71. }
  72. return n, nil
  73. }
  74. func IntMin(a int, b int) int {
  75. if a < b {
  76. return a
  77. }
  78. return b
  79. }