You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

dec_lzma2.go 31KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235
  1. /*
  2. * LZMA2 decoder
  3. *
  4. * Authors: Lasse Collin <lasse.collin@tukaani.org>
  5. * Igor Pavlov <http://7-zip.org/>
  6. *
  7. * Translation to Go: Michael Cross <https://github.com/xi2>
  8. *
  9. * This file has been put into the public domain.
  10. * You can do whatever you want with this file.
  11. */
  12. package xz
  13. /* from linux/lib/xz/xz_lzma2.h ***************************************/
  14. /* Range coder constants */
  15. const (
  16. rcShiftBits = 8
  17. rcTopBits = 24
  18. rcTopValue = 1 << rcTopBits
  19. rcBitModelTotalBits = 11
  20. rcBitModelTotal = 1 << rcBitModelTotalBits
  21. rcMoveBits = 5
  22. )
  23. /*
  24. * Maximum number of position states. A position state is the lowest pb
  25. * number of bits of the current uncompressed offset. In some places there
  26. * are different sets of probabilities for different position states.
  27. */
  28. const posStatesMax = 1 << 4
  29. /*
  30. * lzmaState is used to track which LZMA symbols have occurred most recently
  31. * and in which order. This information is used to predict the next symbol.
  32. *
  33. * Symbols:
  34. * - Literal: One 8-bit byte
  35. * - Match: Repeat a chunk of data at some distance
  36. * - Long repeat: Multi-byte match at a recently seen distance
  37. * - Short repeat: One-byte repeat at a recently seen distance
  38. *
  39. * The symbol names are in from STATE-oldest-older-previous. REP means
  40. * either short or long repeated match, and NONLIT means any non-literal.
  41. */
  42. type lzmaState int
  43. const (
  44. stateLitLit lzmaState = iota
  45. stateMatchLitLit
  46. stateRepLitLit
  47. stateShortrepLitLit
  48. stateMatchLit
  49. stateRepList
  50. stateShortrepLit
  51. stateLitMatch
  52. stateLitLongrep
  53. stateLitShortrep
  54. stateNonlitMatch
  55. stateNonlitRep
  56. )
  57. /* Total number of states */
  58. const states = 12
  59. /* The lowest 7 states indicate that the previous state was a literal. */
  60. const litStates = 7
  61. /* Indicate that the latest symbol was a literal. */
  62. func lzmaStateLiteral(state *lzmaState) {
  63. switch {
  64. case *state <= stateShortrepLitLit:
  65. *state = stateLitLit
  66. case *state <= stateLitShortrep:
  67. *state -= 3
  68. default:
  69. *state -= 6
  70. }
  71. }
  72. /* Indicate that the latest symbol was a match. */
  73. func lzmaStateMatch(state *lzmaState) {
  74. if *state < litStates {
  75. *state = stateLitMatch
  76. } else {
  77. *state = stateNonlitMatch
  78. }
  79. }
  80. /* Indicate that the latest state was a long repeated match. */
  81. func lzmaStateLongRep(state *lzmaState) {
  82. if *state < litStates {
  83. *state = stateLitLongrep
  84. } else {
  85. *state = stateNonlitRep
  86. }
  87. }
  88. /* Indicate that the latest symbol was a short match. */
  89. func lzmaStateShortRep(state *lzmaState) {
  90. if *state < litStates {
  91. *state = stateLitShortrep
  92. } else {
  93. *state = stateNonlitRep
  94. }
  95. }
  96. /* Test if the previous symbol was a literal. */
  97. func lzmaStateIsLiteral(state lzmaState) bool {
  98. return state < litStates
  99. }
  100. /* Each literal coder is divided in three sections:
  101. * - 0x001-0x0FF: Without match byte
  102. * - 0x101-0x1FF: With match byte; match bit is 0
  103. * - 0x201-0x2FF: With match byte; match bit is 1
  104. *
  105. * Match byte is used when the previous LZMA symbol was something else than
  106. * a literal (that is, it was some kind of match).
  107. */
  108. const literalCoderSize = 0x300
  109. /* Maximum number of literal coders */
  110. const literalCodersMax = 1 << 4
  111. /* Minimum length of a match is two bytes. */
  112. const matchLenMin = 2
  113. /* Match length is encoded with 4, 5, or 10 bits.
  114. *
  115. * Length Bits
  116. * 2-9 4 = Choice=0 + 3 bits
  117. * 10-17 5 = Choice=1 + Choice2=0 + 3 bits
  118. * 18-273 10 = Choice=1 + Choice2=1 + 8 bits
  119. */
  120. const (
  121. lenLowBits = 3
  122. lenLowSymbols = 1 << lenLowBits
  123. lenMidBits = 3
  124. lenMidSymbols = 1 << lenMidBits
  125. lenHighBits = 8
  126. lenHighSymbols = 1 << lenHighBits
  127. )
  128. /*
  129. * Different sets of probabilities are used for match distances that have
  130. * very short match length: Lengths of 2, 3, and 4 bytes have a separate
  131. * set of probabilities for each length. The matches with longer length
  132. * use a shared set of probabilities.
  133. */
  134. const distStates = 4
  135. /*
  136. * Get the index of the appropriate probability array for decoding
  137. * the distance slot.
  138. */
  139. func lzmaGetDistState(len uint32) uint32 {
  140. if len < distStates+matchLenMin {
  141. return len - matchLenMin
  142. } else {
  143. return distStates - 1
  144. }
  145. }
  146. /*
  147. * The highest two bits of a 32-bit match distance are encoded using six bits.
  148. * This six-bit value is called a distance slot. This way encoding a 32-bit
  149. * value takes 6-36 bits, larger values taking more bits.
  150. */
  151. const (
  152. distSlotBits = 6
  153. distSlots = 1 << distSlotBits
  154. )
  155. /* Match distances up to 127 are fully encoded using probabilities. Since
  156. * the highest two bits (distance slot) are always encoded using six bits,
  157. * the distances 0-3 don't need any additional bits to encode, since the
  158. * distance slot itself is the same as the actual distance. distModelStart
  159. * indicates the first distance slot where at least one additional bit is
  160. * needed.
  161. */
  162. const distModelStart = 4
  163. /*
  164. * Match distances greater than 127 are encoded in three pieces:
  165. * - distance slot: the highest two bits
  166. * - direct bits: 2-26 bits below the highest two bits
  167. * - alignment bits: four lowest bits
  168. *
  169. * Direct bits don't use any probabilities.
  170. *
  171. * The distance slot value of 14 is for distances 128-191.
  172. */
  173. const distModelEnd = 14
  174. /* Distance slots that indicate a distance <= 127. */
  175. const (
  176. fullDistancesBits = distModelEnd / 2
  177. fullDistances = 1 << fullDistancesBits
  178. )
  179. /*
  180. * For match distances greater than 127, only the highest two bits and the
  181. * lowest four bits (alignment) is encoded using probabilities.
  182. */
  183. const (
  184. alignBits = 4
  185. alignSize = 1 << alignBits
  186. )
  187. /* from linux/lib/xz/xz_dec_lzma2.c ***********************************/
  188. /*
  189. * Range decoder initialization eats the first five bytes of each LZMA chunk.
  190. */
  191. const rcInitBytes = 5
  192. /*
  193. * Minimum number of usable input buffer to safely decode one LZMA symbol.
  194. * The worst case is that we decode 22 bits using probabilities and 26
  195. * direct bits. This may decode at maximum of 20 bytes of input. However,
  196. * lzmaMain does an extra normalization before returning, thus we
  197. * need to put 21 here.
  198. */
  199. const lzmaInRequired = 21
  200. /*
  201. * Dictionary (history buffer)
  202. *
  203. * These are always true:
  204. * start <= pos <= full <= end
  205. * pos <= limit <= end
  206. * end == size
  207. * size <= sizeMax
  208. * len(buf) <= size
  209. */
  210. type dictionary struct {
  211. /* The history buffer */
  212. buf []byte
  213. /* Old position in buf (before decoding more data) */
  214. start uint32
  215. /* Position in buf */
  216. pos uint32
  217. /*
  218. * How full dictionary is. This is used to detect corrupt input that
  219. * would read beyond the beginning of the uncompressed stream.
  220. */
  221. full uint32
  222. /* Write limit; we don't write to buf[limit] or later bytes. */
  223. limit uint32
  224. /*
  225. * End of the dictionary buffer. This is the same as the
  226. * dictionary size.
  227. */
  228. end uint32
  229. /*
  230. * Size of the dictionary as specified in Block Header. This is used
  231. * together with "full" to detect corrupt input that would make us
  232. * read beyond the beginning of the uncompressed stream.
  233. */
  234. size uint32
  235. /* Maximum allowed dictionary size. */
  236. sizeMax uint32
  237. }
  238. /* Range decoder */
  239. type rcDec struct {
  240. rnge uint32
  241. code uint32
  242. /*
  243. * Number of initializing bytes remaining to be read
  244. * by rcReadInit.
  245. */
  246. initBytesLeft uint32
  247. /*
  248. * Buffer from which we read our input. It can be either
  249. * temp.buf or the caller-provided input buffer.
  250. */
  251. in []byte
  252. inPos int
  253. inLimit int
  254. }
  255. /* Probabilities for a length decoder. */
  256. type lzmaLenDec struct {
  257. /* Probability of match length being at least 10 */
  258. choice uint16
  259. /* Probability of match length being at least 18 */
  260. choice2 uint16
  261. /* Probabilities for match lengths 2-9 */
  262. low [posStatesMax][lenLowSymbols]uint16
  263. /* Probabilities for match lengths 10-17 */
  264. mid [posStatesMax][lenMidSymbols]uint16
  265. /* Probabilities for match lengths 18-273 */
  266. high [lenHighSymbols]uint16
  267. }
  268. type lzmaDec struct {
  269. /* Distances of latest four matches */
  270. rep0 uint32
  271. rep1 uint32
  272. rep2 uint32
  273. rep3 uint32
  274. /* Types of the most recently seen LZMA symbols */
  275. state lzmaState
  276. /*
  277. * Length of a match. This is updated so that dictRepeat can
  278. * be called again to finish repeating the whole match.
  279. */
  280. len uint32
  281. /*
  282. * LZMA properties or related bit masks (number of literal
  283. * context bits, a mask derived from the number of literal
  284. * position bits, and a mask derived from the number
  285. * position bits)
  286. */
  287. lc uint32
  288. literalPosMask uint32
  289. posMask uint32
  290. /* If 1, it's a match. Otherwise it's a single 8-bit literal. */
  291. isMatch [states][posStatesMax]uint16
  292. /* If 1, it's a repeated match. The distance is one of rep0 .. rep3. */
  293. isRep [states]uint16
  294. /*
  295. * If 0, distance of a repeated match is rep0.
  296. * Otherwise check is_rep1.
  297. */
  298. isRep0 [states]uint16
  299. /*
  300. * If 0, distance of a repeated match is rep1.
  301. * Otherwise check is_rep2.
  302. */
  303. isRep1 [states]uint16
  304. /* If 0, distance of a repeated match is rep2. Otherwise it is rep3. */
  305. isRep2 [states]uint16
  306. /*
  307. * If 1, the repeated match has length of one byte. Otherwise
  308. * the length is decoded from rep_len_decoder.
  309. */
  310. isRep0Long [states][posStatesMax]uint16
  311. /*
  312. * Probability tree for the highest two bits of the match
  313. * distance. There is a separate probability tree for match
  314. * lengths of 2 (i.e. MATCH_LEN_MIN), 3, 4, and [5, 273].
  315. */
  316. distSlot [distStates][distSlots]uint16
  317. /*
  318. * Probility trees for additional bits for match distance
  319. * when the distance is in the range [4, 127].
  320. */
  321. distSpecial [fullDistances - distModelEnd]uint16
  322. /*
  323. * Probability tree for the lowest four bits of a match
  324. * distance that is equal to or greater than 128.
  325. */
  326. distAlign [alignSize]uint16
  327. /* Length of a normal match */
  328. matchLenDec lzmaLenDec
  329. /* Length of a repeated match */
  330. repLenDec lzmaLenDec
  331. /* Probabilities of literals */
  332. literal [literalCodersMax][literalCoderSize]uint16
  333. }
  334. // type of lzma2Dec.sequence
  335. type lzma2Seq int
  336. const (
  337. seqControl lzma2Seq = iota
  338. seqUncompressed1
  339. seqUncompressed2
  340. seqCompressed0
  341. seqCompressed1
  342. seqProperties
  343. seqLZMAPrepare
  344. seqLZMARun
  345. seqCopy
  346. )
  347. type lzma2Dec struct {
  348. /* Position in xzDecLZMA2Run. */
  349. sequence lzma2Seq
  350. /* Next position after decoding the compressed size of the chunk. */
  351. nextSequence lzma2Seq
  352. /* Uncompressed size of LZMA chunk (2 MiB at maximum) */
  353. uncompressed int
  354. /*
  355. * Compressed size of LZMA chunk or compressed/uncompressed
  356. * size of uncompressed chunk (64 KiB at maximum)
  357. */
  358. compressed int
  359. /*
  360. * True if dictionary reset is needed. This is false before
  361. * the first chunk (LZMA or uncompressed).
  362. */
  363. needDictReset bool
  364. /*
  365. * True if new LZMA properties are needed. This is false
  366. * before the first LZMA chunk.
  367. */
  368. needProps bool
  369. }
  370. type xzDecLZMA2 struct {
  371. /*
  372. * The order below is important on x86 to reduce code size and
  373. * it shouldn't hurt on other platforms. Everything up to and
  374. * including lzma.pos_mask are in the first 128 bytes on x86-32,
  375. * which allows using smaller instructions to access those
  376. * variables. On x86-64, fewer variables fit into the first 128
  377. * bytes, but this is still the best order without sacrificing
  378. * the readability by splitting the structures.
  379. */
  380. rc rcDec
  381. dict dictionary
  382. lzma2 lzma2Dec
  383. lzma lzmaDec
  384. /*
  385. * Temporary buffer which holds small number of input bytes between
  386. * decoder calls. See lzma2LZMA for details.
  387. */
  388. temp struct {
  389. buf []byte // slice buf will be backed by bufArray
  390. bufArray [3 * lzmaInRequired]byte
  391. }
  392. }
  393. /**************
  394. * Dictionary *
  395. **************/
  396. /*
  397. * Reset the dictionary state. When in single-call mode, set up the beginning
  398. * of the dictionary to point to the actual output buffer.
  399. */
  400. func dictReset(dict *dictionary, b *xzBuf) {
  401. dict.start = 0
  402. dict.pos = 0
  403. dict.limit = 0
  404. dict.full = 0
  405. }
  406. /* Set dictionary write limit */
  407. func dictLimit(dict *dictionary, outMax int) {
  408. if dict.end-dict.pos <= uint32(outMax) {
  409. dict.limit = dict.end
  410. } else {
  411. dict.limit = dict.pos + uint32(outMax)
  412. }
  413. }
  414. /* Return true if at least one byte can be written into the dictionary. */
  415. func dictHasSpace(dict *dictionary) bool {
  416. return dict.pos < dict.limit
  417. }
  418. /*
  419. * Get a byte from the dictionary at the given distance. The distance is
  420. * assumed to valid, or as a special case, zero when the dictionary is
  421. * still empty. This special case is needed for single-call decoding to
  422. * avoid writing a '\x00' to the end of the destination buffer.
  423. */
  424. func dictGet(dict *dictionary, dist uint32) uint32 {
  425. var offset uint32 = dict.pos - dist - 1
  426. if dist >= dict.pos {
  427. offset += dict.end
  428. }
  429. if dict.full > 0 {
  430. return uint32(dict.buf[offset])
  431. }
  432. return 0
  433. }
  434. /*
  435. * Put one byte into the dictionary. It is assumed that there is space for it.
  436. */
  437. func dictPut(dict *dictionary, byte byte) {
  438. dict.buf[dict.pos] = byte
  439. dict.pos++
  440. if dict.full < dict.pos {
  441. dict.full = dict.pos
  442. }
  443. }
  444. /*
  445. * Repeat given number of bytes from the given distance. If the distance is
  446. * invalid, false is returned. On success, true is returned and *len is
  447. * updated to indicate how many bytes were left to be repeated.
  448. */
  449. func dictRepeat(dict *dictionary, len *uint32, dist uint32) bool {
  450. var back uint32
  451. var left uint32
  452. if dist >= dict.full || dist >= dict.size {
  453. return false
  454. }
  455. left = dict.limit - dict.pos
  456. if left > *len {
  457. left = *len
  458. }
  459. *len -= left
  460. back = dict.pos - dist - 1
  461. if dist >= dict.pos {
  462. back += dict.end
  463. }
  464. for {
  465. dict.buf[dict.pos] = dict.buf[back]
  466. dict.pos++
  467. back++
  468. if back == dict.end {
  469. back = 0
  470. }
  471. left--
  472. if !(left > 0) {
  473. break
  474. }
  475. }
  476. if dict.full < dict.pos {
  477. dict.full = dict.pos
  478. }
  479. return true
  480. }
  481. /* Copy uncompressed data as is from input to dictionary and output buffers. */
  482. func dictUncompressed(dict *dictionary, b *xzBuf, left *int) {
  483. var copySize int
  484. for *left > 0 && b.inPos < len(b.in) && b.outPos < len(b.out) {
  485. copySize = len(b.in) - b.inPos
  486. if copySize > len(b.out)-b.outPos {
  487. copySize = len(b.out) - b.outPos
  488. }
  489. if copySize > int(dict.end-dict.pos) {
  490. copySize = int(dict.end - dict.pos)
  491. }
  492. if copySize > *left {
  493. copySize = *left
  494. }
  495. *left -= copySize
  496. copy(dict.buf[dict.pos:], b.in[b.inPos:b.inPos+copySize])
  497. dict.pos += uint32(copySize)
  498. if dict.full < dict.pos {
  499. dict.full = dict.pos
  500. }
  501. if dict.pos == dict.end {
  502. dict.pos = 0
  503. }
  504. copy(b.out[b.outPos:], b.in[b.inPos:b.inPos+copySize])
  505. dict.start = dict.pos
  506. b.outPos += copySize
  507. b.inPos += copySize
  508. }
  509. }
  510. /*
  511. * Flush pending data from dictionary to b.out. It is assumed that there is
  512. * enough space in b.out. This is guaranteed because caller uses dictLimit
  513. * before decoding data into the dictionary.
  514. */
  515. func dictFlush(dict *dictionary, b *xzBuf) int {
  516. var copySize int = int(dict.pos - dict.start)
  517. if dict.pos == dict.end {
  518. dict.pos = 0
  519. }
  520. copy(b.out[b.outPos:], dict.buf[dict.start:dict.start+uint32(copySize)])
  521. dict.start = dict.pos
  522. b.outPos += copySize
  523. return copySize
  524. }
  525. /*****************
  526. * Range decoder *
  527. *****************/
  528. /* Reset the range decoder. */
  529. func rcReset(rc *rcDec) {
  530. rc.rnge = ^uint32(0)
  531. rc.code = 0
  532. rc.initBytesLeft = rcInitBytes
  533. }
  534. /*
  535. * Read the first five initial bytes into rc->code if they haven't been
  536. * read already. (Yes, the first byte gets completely ignored.)
  537. */
  538. func rcReadInit(rc *rcDec, b *xzBuf) bool {
  539. for rc.initBytesLeft > 0 {
  540. if b.inPos == len(b.in) {
  541. return false
  542. }
  543. rc.code = rc.code<<8 + uint32(b.in[b.inPos])
  544. b.inPos++
  545. rc.initBytesLeft--
  546. }
  547. return true
  548. }
  549. /* Return true if there may not be enough input for the next decoding loop. */
  550. func rcLimitExceeded(rc *rcDec) bool {
  551. return rc.inPos > rc.inLimit
  552. }
  553. /*
  554. * Return true if it is possible (from point of view of range decoder) that
  555. * we have reached the end of the LZMA chunk.
  556. */
  557. func rcIsFinished(rc *rcDec) bool {
  558. return rc.code == 0
  559. }
  560. /* Read the next input byte if needed. */
  561. func rcNormalize(rc *rcDec) {
  562. if rc.rnge < rcTopValue {
  563. rc.rnge <<= rcShiftBits
  564. rc.code = rc.code<<rcShiftBits + uint32(rc.in[rc.inPos])
  565. rc.inPos++
  566. }
  567. }
  568. /* Decode one bit. */
  569. func rcBit(rc *rcDec, prob *uint16) bool {
  570. var bound uint32
  571. var bit bool
  572. rcNormalize(rc)
  573. bound = (rc.rnge >> rcBitModelTotalBits) * uint32(*prob)
  574. if rc.code < bound {
  575. rc.rnge = bound
  576. *prob += (rcBitModelTotal - *prob) >> rcMoveBits
  577. bit = false
  578. } else {
  579. rc.rnge -= bound
  580. rc.code -= bound
  581. *prob -= *prob >> rcMoveBits
  582. bit = true
  583. }
  584. return bit
  585. }
  586. /* Decode a bittree starting from the most significant bit. */
  587. func rcBittree(rc *rcDec, probs []uint16, limit uint32) uint32 {
  588. var symbol uint32 = 1
  589. for {
  590. if rcBit(rc, &probs[symbol-1]) {
  591. symbol = symbol<<1 + 1
  592. } else {
  593. symbol <<= 1
  594. }
  595. if !(symbol < limit) {
  596. break
  597. }
  598. }
  599. return symbol
  600. }
  601. /* Decode a bittree starting from the least significant bit. */
  602. func rcBittreeReverse(rc *rcDec, probs []uint16, dest *uint32, limit uint32) {
  603. var symbol uint32 = 1
  604. var i uint32 = 0
  605. for {
  606. if rcBit(rc, &probs[symbol-1]) {
  607. symbol = symbol<<1 + 1
  608. *dest += 1 << i
  609. } else {
  610. symbol <<= 1
  611. }
  612. i++
  613. if !(i < limit) {
  614. break
  615. }
  616. }
  617. }
  618. /* Decode direct bits (fixed fifty-fifty probability) */
  619. func rcDirect(rc *rcDec, dest *uint32, limit uint32) {
  620. var mask uint32
  621. for {
  622. rcNormalize(rc)
  623. rc.rnge >>= 1
  624. rc.code -= rc.rnge
  625. mask = 0 - rc.code>>31
  626. rc.code += rc.rnge & mask
  627. *dest = *dest<<1 + mask + 1
  628. limit--
  629. if !(limit > 0) {
  630. break
  631. }
  632. }
  633. }
  634. /********
  635. * LZMA *
  636. ********/
  637. /* Get pointer to literal coder probability array. */
  638. func lzmaLiteralProbs(s *xzDecLZMA2) []uint16 {
  639. var prevByte uint32 = dictGet(&s.dict, 0)
  640. var low uint32 = prevByte >> (8 - s.lzma.lc)
  641. var high uint32 = (s.dict.pos & s.lzma.literalPosMask) << s.lzma.lc
  642. return s.lzma.literal[low+high][:]
  643. }
  644. /* Decode a literal (one 8-bit byte) */
  645. func lzmaLiteral(s *xzDecLZMA2) {
  646. var probs []uint16
  647. var symbol uint32
  648. var matchByte uint32
  649. var matchBit uint32
  650. var offset uint32
  651. var i uint32
  652. probs = lzmaLiteralProbs(s)
  653. if lzmaStateIsLiteral(s.lzma.state) {
  654. symbol = rcBittree(&s.rc, probs[1:], 0x100)
  655. } else {
  656. symbol = 1
  657. matchByte = dictGet(&s.dict, s.lzma.rep0) << 1
  658. offset = 0x100
  659. for {
  660. matchBit = matchByte & offset
  661. matchByte <<= 1
  662. i = offset + matchBit + symbol
  663. if rcBit(&s.rc, &probs[i]) {
  664. symbol = symbol<<1 + 1
  665. offset &= matchBit
  666. } else {
  667. symbol <<= 1
  668. offset &= ^matchBit
  669. }
  670. if !(symbol < 0x100) {
  671. break
  672. }
  673. }
  674. }
  675. dictPut(&s.dict, byte(symbol))
  676. lzmaStateLiteral(&s.lzma.state)
  677. }
  678. /* Decode the length of the match into s.lzma.len. */
  679. func lzmaLen(s *xzDecLZMA2, l *lzmaLenDec, posState uint32) {
  680. var probs []uint16
  681. var limit uint32
  682. switch {
  683. case !rcBit(&s.rc, &l.choice):
  684. probs = l.low[posState][:]
  685. limit = lenLowSymbols
  686. s.lzma.len = matchLenMin
  687. case !rcBit(&s.rc, &l.choice2):
  688. probs = l.mid[posState][:]
  689. limit = lenMidSymbols
  690. s.lzma.len = matchLenMin + lenLowSymbols
  691. default:
  692. probs = l.high[:]
  693. limit = lenHighSymbols
  694. s.lzma.len = matchLenMin + lenLowSymbols + lenMidSymbols
  695. }
  696. s.lzma.len += rcBittree(&s.rc, probs[1:], limit) - limit
  697. }
  698. /* Decode a match. The distance will be stored in s.lzma.rep0. */
  699. func lzmaMatch(s *xzDecLZMA2, posState uint32) {
  700. var probs []uint16
  701. var distSlot uint32
  702. var limit uint32
  703. lzmaStateMatch(&s.lzma.state)
  704. s.lzma.rep3 = s.lzma.rep2
  705. s.lzma.rep2 = s.lzma.rep1
  706. s.lzma.rep1 = s.lzma.rep0
  707. lzmaLen(s, &s.lzma.matchLenDec, posState)
  708. probs = s.lzma.distSlot[lzmaGetDistState(s.lzma.len)][:]
  709. distSlot = rcBittree(&s.rc, probs[1:], distSlots) - distSlots
  710. if distSlot < distModelStart {
  711. s.lzma.rep0 = distSlot
  712. } else {
  713. limit = distSlot>>1 - 1
  714. s.lzma.rep0 = 2 + distSlot&1
  715. if distSlot < distModelEnd {
  716. s.lzma.rep0 <<= limit
  717. probs = s.lzma.distSpecial[s.lzma.rep0-distSlot:]
  718. rcBittreeReverse(&s.rc, probs, &s.lzma.rep0, limit)
  719. } else {
  720. rcDirect(&s.rc, &s.lzma.rep0, limit-alignBits)
  721. s.lzma.rep0 <<= alignBits
  722. rcBittreeReverse(
  723. &s.rc, s.lzma.distAlign[1:], &s.lzma.rep0, alignBits)
  724. }
  725. }
  726. }
  727. /*
  728. * Decode a repeated match. The distance is one of the four most recently
  729. * seen matches. The distance will be stored in s.lzma.rep0.
  730. */
  731. func lzmaRepMatch(s *xzDecLZMA2, posState uint32) {
  732. var tmp uint32
  733. if !rcBit(&s.rc, &s.lzma.isRep0[s.lzma.state]) {
  734. if !rcBit(&s.rc, &s.lzma.isRep0Long[s.lzma.state][posState]) {
  735. lzmaStateShortRep(&s.lzma.state)
  736. s.lzma.len = 1
  737. return
  738. }
  739. } else {
  740. if !rcBit(&s.rc, &s.lzma.isRep1[s.lzma.state]) {
  741. tmp = s.lzma.rep1
  742. } else {
  743. if !rcBit(&s.rc, &s.lzma.isRep2[s.lzma.state]) {
  744. tmp = s.lzma.rep2
  745. } else {
  746. tmp = s.lzma.rep3
  747. s.lzma.rep3 = s.lzma.rep2
  748. }
  749. s.lzma.rep2 = s.lzma.rep1
  750. }
  751. s.lzma.rep1 = s.lzma.rep0
  752. s.lzma.rep0 = tmp
  753. }
  754. lzmaStateLongRep(&s.lzma.state)
  755. lzmaLen(s, &s.lzma.repLenDec, posState)
  756. }
  757. /* LZMA decoder core */
  758. func lzmaMain(s *xzDecLZMA2) bool {
  759. var posState uint32
  760. /*
  761. * If the dictionary was reached during the previous call, try to
  762. * finish the possibly pending repeat in the dictionary.
  763. */
  764. if dictHasSpace(&s.dict) && s.lzma.len > 0 {
  765. dictRepeat(&s.dict, &s.lzma.len, s.lzma.rep0)
  766. }
  767. /*
  768. * Decode more LZMA symbols. One iteration may consume up to
  769. * lzmaInRequired - 1 bytes.
  770. */
  771. for dictHasSpace(&s.dict) && !rcLimitExceeded(&s.rc) {
  772. posState = s.dict.pos & s.lzma.posMask
  773. if !rcBit(&s.rc, &s.lzma.isMatch[s.lzma.state][posState]) {
  774. lzmaLiteral(s)
  775. } else {
  776. if rcBit(&s.rc, &s.lzma.isRep[s.lzma.state]) {
  777. lzmaRepMatch(s, posState)
  778. } else {
  779. lzmaMatch(s, posState)
  780. }
  781. if !dictRepeat(&s.dict, &s.lzma.len, s.lzma.rep0) {
  782. return false
  783. }
  784. }
  785. }
  786. /*
  787. * Having the range decoder always normalized when we are outside
  788. * this function makes it easier to correctly handle end of the chunk.
  789. */
  790. rcNormalize(&s.rc)
  791. return true
  792. }
  793. /*
  794. * Reset the LZMA decoder and range decoder state. Dictionary is not reset
  795. * here, because LZMA state may be reset without resetting the dictionary.
  796. */
  797. func lzmaReset(s *xzDecLZMA2) {
  798. s.lzma.state = stateLitLit
  799. s.lzma.rep0 = 0
  800. s.lzma.rep1 = 0
  801. s.lzma.rep2 = 0
  802. s.lzma.rep3 = 0
  803. /* All probabilities are initialized to the same value, v */
  804. v := uint16(rcBitModelTotal / 2)
  805. s.lzma.matchLenDec.choice = v
  806. s.lzma.matchLenDec.choice2 = v
  807. s.lzma.repLenDec.choice = v
  808. s.lzma.repLenDec.choice2 = v
  809. for _, m := range [][]uint16{
  810. s.lzma.isRep[:], s.lzma.isRep0[:], s.lzma.isRep1[:],
  811. s.lzma.isRep2[:], s.lzma.distSpecial[:], s.lzma.distAlign[:],
  812. s.lzma.matchLenDec.high[:], s.lzma.repLenDec.high[:],
  813. } {
  814. for j := range m {
  815. m[j] = v
  816. }
  817. }
  818. for i := range s.lzma.isMatch {
  819. for j := range s.lzma.isMatch[i] {
  820. s.lzma.isMatch[i][j] = v
  821. }
  822. }
  823. for i := range s.lzma.isRep0Long {
  824. for j := range s.lzma.isRep0Long[i] {
  825. s.lzma.isRep0Long[i][j] = v
  826. }
  827. }
  828. for i := range s.lzma.distSlot {
  829. for j := range s.lzma.distSlot[i] {
  830. s.lzma.distSlot[i][j] = v
  831. }
  832. }
  833. for i := range s.lzma.literal {
  834. for j := range s.lzma.literal[i] {
  835. s.lzma.literal[i][j] = v
  836. }
  837. }
  838. for i := range s.lzma.matchLenDec.low {
  839. for j := range s.lzma.matchLenDec.low[i] {
  840. s.lzma.matchLenDec.low[i][j] = v
  841. }
  842. }
  843. for i := range s.lzma.matchLenDec.mid {
  844. for j := range s.lzma.matchLenDec.mid[i] {
  845. s.lzma.matchLenDec.mid[i][j] = v
  846. }
  847. }
  848. for i := range s.lzma.repLenDec.low {
  849. for j := range s.lzma.repLenDec.low[i] {
  850. s.lzma.repLenDec.low[i][j] = v
  851. }
  852. }
  853. for i := range s.lzma.repLenDec.mid {
  854. for j := range s.lzma.repLenDec.mid[i] {
  855. s.lzma.repLenDec.mid[i][j] = v
  856. }
  857. }
  858. rcReset(&s.rc)
  859. }
  860. /*
  861. * Decode and validate LZMA properties (lc/lp/pb) and calculate the bit masks
  862. * from the decoded lp and pb values. On success, the LZMA decoder state is
  863. * reset and true is returned.
  864. */
  865. func lzmaProps(s *xzDecLZMA2, props byte) bool {
  866. if props > (4*5+4)*9+8 {
  867. return false
  868. }
  869. s.lzma.posMask = 0
  870. for props >= 9*5 {
  871. props -= 9 * 5
  872. s.lzma.posMask++
  873. }
  874. s.lzma.posMask = 1<<s.lzma.posMask - 1
  875. s.lzma.literalPosMask = 0
  876. for props >= 9 {
  877. props -= 9
  878. s.lzma.literalPosMask++
  879. }
  880. s.lzma.lc = uint32(props)
  881. if s.lzma.lc+s.lzma.literalPosMask > 4 {
  882. return false
  883. }
  884. s.lzma.literalPosMask = 1<<s.lzma.literalPosMask - 1
  885. lzmaReset(s)
  886. return true
  887. }
  888. /*********
  889. * LZMA2 *
  890. *********/
  891. /*
  892. * The LZMA decoder assumes that if the input limit (s.rc.inLimit) hasn't
  893. * been exceeded, it is safe to read up to lzmaInRequired bytes. This
  894. * wrapper function takes care of making the LZMA decoder's assumption safe.
  895. *
  896. * As long as there is plenty of input left to be decoded in the current LZMA
  897. * chunk, we decode directly from the caller-supplied input buffer until
  898. * there's lzmaInRequired bytes left. Those remaining bytes are copied into
  899. * s.temp.buf, which (hopefully) gets filled on the next call to this
  900. * function. We decode a few bytes from the temporary buffer so that we can
  901. * continue decoding from the caller-supplied input buffer again.
  902. */
  903. func lzma2LZMA(s *xzDecLZMA2, b *xzBuf) bool {
  904. var inAvail int
  905. var tmp int
  906. inAvail = len(b.in) - b.inPos
  907. if len(s.temp.buf) > 0 || s.lzma2.compressed == 0 {
  908. tmp = 2*lzmaInRequired - len(s.temp.buf)
  909. if tmp > s.lzma2.compressed-len(s.temp.buf) {
  910. tmp = s.lzma2.compressed - len(s.temp.buf)
  911. }
  912. if tmp > inAvail {
  913. tmp = inAvail
  914. }
  915. copy(s.temp.bufArray[len(s.temp.buf):], b.in[b.inPos:b.inPos+tmp])
  916. switch {
  917. case len(s.temp.buf)+tmp == s.lzma2.compressed:
  918. for i := len(s.temp.buf) + tmp; i < len(s.temp.bufArray); i++ {
  919. s.temp.bufArray[i] = 0
  920. }
  921. s.rc.inLimit = len(s.temp.buf) + tmp
  922. case len(s.temp.buf)+tmp < lzmaInRequired:
  923. s.temp.buf = s.temp.bufArray[:len(s.temp.buf)+tmp]
  924. b.inPos += tmp
  925. return true
  926. default:
  927. s.rc.inLimit = len(s.temp.buf) + tmp - lzmaInRequired
  928. }
  929. s.rc.in = s.temp.bufArray[:]
  930. s.rc.inPos = 0
  931. if !lzmaMain(s) || s.rc.inPos > len(s.temp.buf)+tmp {
  932. return false
  933. }
  934. s.lzma2.compressed -= s.rc.inPos
  935. if s.rc.inPos < len(s.temp.buf) {
  936. copy(s.temp.buf, s.temp.buf[s.rc.inPos:])
  937. s.temp.buf = s.temp.buf[:len(s.temp.buf)-s.rc.inPos]
  938. return true
  939. }
  940. b.inPos += s.rc.inPos - len(s.temp.buf)
  941. s.temp.buf = nil
  942. }
  943. inAvail = len(b.in) - b.inPos
  944. if inAvail >= lzmaInRequired {
  945. s.rc.in = b.in
  946. s.rc.inPos = b.inPos
  947. if inAvail >= s.lzma2.compressed+lzmaInRequired {
  948. s.rc.inLimit = b.inPos + s.lzma2.compressed
  949. } else {
  950. s.rc.inLimit = len(b.in) - lzmaInRequired
  951. }
  952. if !lzmaMain(s) {
  953. return false
  954. }
  955. inAvail = s.rc.inPos - b.inPos
  956. if inAvail > s.lzma2.compressed {
  957. return false
  958. }
  959. s.lzma2.compressed -= inAvail
  960. b.inPos = s.rc.inPos
  961. }
  962. inAvail = len(b.in) - b.inPos
  963. if inAvail < lzmaInRequired {
  964. if inAvail > s.lzma2.compressed {
  965. inAvail = s.lzma2.compressed
  966. }
  967. s.temp.buf = s.temp.bufArray[:inAvail]
  968. copy(s.temp.buf, b.in[b.inPos:])
  969. b.inPos += inAvail
  970. }
  971. return true
  972. }
  973. /*
  974. * Take care of the LZMA2 control layer, and forward the job of actual LZMA
  975. * decoding or copying of uncompressed chunks to other functions.
  976. */
  977. func xzDecLZMA2Run(s *xzDecLZMA2, b *xzBuf) xzRet {
  978. var tmp int
  979. for b.inPos < len(b.in) || s.lzma2.sequence == seqLZMARun {
  980. switch s.lzma2.sequence {
  981. case seqControl:
  982. /*
  983. * LZMA2 control byte
  984. *
  985. * Exact values:
  986. * 0x00 End marker
  987. * 0x01 Dictionary reset followed by
  988. * an uncompressed chunk
  989. * 0x02 Uncompressed chunk (no dictionary reset)
  990. *
  991. * Highest three bits (s.control & 0xE0):
  992. * 0xE0 Dictionary reset, new properties and state
  993. * reset, followed by LZMA compressed chunk
  994. * 0xC0 New properties and state reset, followed
  995. * by LZMA compressed chunk (no dictionary
  996. * reset)
  997. * 0xA0 State reset using old properties,
  998. * followed by LZMA compressed chunk (no
  999. * dictionary reset)
  1000. * 0x80 LZMA chunk (no dictionary or state reset)
  1001. *
  1002. * For LZMA compressed chunks, the lowest five bits
  1003. * (s.control & 1F) are the highest bits of the
  1004. * uncompressed size (bits 16-20).
  1005. *
  1006. * A new LZMA2 stream must begin with a dictionary
  1007. * reset. The first LZMA chunk must set new
  1008. * properties and reset the LZMA state.
  1009. *
  1010. * Values that don't match anything described above
  1011. * are invalid and we return xzDataError.
  1012. */
  1013. tmp = int(b.in[b.inPos])
  1014. b.inPos++
  1015. if tmp == 0x00 {
  1016. return xzStreamEnd
  1017. }
  1018. switch {
  1019. case tmp >= 0xe0 || tmp == 0x01:
  1020. s.lzma2.needProps = true
  1021. s.lzma2.needDictReset = false
  1022. dictReset(&s.dict, b)
  1023. case s.lzma2.needDictReset:
  1024. return xzDataError
  1025. }
  1026. if tmp >= 0x80 {
  1027. s.lzma2.uncompressed = (tmp & 0x1f) << 16
  1028. s.lzma2.sequence = seqUncompressed1
  1029. switch {
  1030. case tmp >= 0xc0:
  1031. /*
  1032. * When there are new properties,
  1033. * state reset is done at
  1034. * seqProperties.
  1035. */
  1036. s.lzma2.needProps = false
  1037. s.lzma2.nextSequence = seqProperties
  1038. case s.lzma2.needProps:
  1039. return xzDataError
  1040. default:
  1041. s.lzma2.nextSequence = seqLZMAPrepare
  1042. if tmp >= 0xa0 {
  1043. lzmaReset(s)
  1044. }
  1045. }
  1046. } else {
  1047. if tmp > 0x02 {
  1048. return xzDataError
  1049. }
  1050. s.lzma2.sequence = seqCompressed0
  1051. s.lzma2.nextSequence = seqCopy
  1052. }
  1053. case seqUncompressed1:
  1054. s.lzma2.uncompressed += int(b.in[b.inPos]) << 8
  1055. b.inPos++
  1056. s.lzma2.sequence = seqUncompressed2
  1057. case seqUncompressed2:
  1058. s.lzma2.uncompressed += int(b.in[b.inPos]) + 1
  1059. b.inPos++
  1060. s.lzma2.sequence = seqCompressed0
  1061. case seqCompressed0:
  1062. s.lzma2.compressed += int(b.in[b.inPos]) << 8
  1063. b.inPos++
  1064. s.lzma2.sequence = seqCompressed1
  1065. case seqCompressed1:
  1066. s.lzma2.compressed += int(b.in[b.inPos]) + 1
  1067. b.inPos++
  1068. s.lzma2.sequence = s.lzma2.nextSequence
  1069. case seqProperties:
  1070. if !lzmaProps(s, b.in[b.inPos]) {
  1071. return xzDataError
  1072. }
  1073. b.inPos++
  1074. s.lzma2.sequence = seqLZMAPrepare
  1075. fallthrough
  1076. case seqLZMAPrepare:
  1077. if s.lzma2.compressed < rcInitBytes {
  1078. return xzDataError
  1079. }
  1080. if !rcReadInit(&s.rc, b) {
  1081. return xzOK
  1082. }
  1083. s.lzma2.compressed -= rcInitBytes
  1084. s.lzma2.sequence = seqLZMARun
  1085. fallthrough
  1086. case seqLZMARun:
  1087. /*
  1088. * Set dictionary limit to indicate how much we want
  1089. * to be encoded at maximum. Decode new data into the
  1090. * dictionary. Flush the new data from dictionary to
  1091. * b.out. Check if we finished decoding this chunk.
  1092. * In case the dictionary got full but we didn't fill
  1093. * the output buffer yet, we may run this loop
  1094. * multiple times without changing s.lzma2.sequence.
  1095. */
  1096. outMax := len(b.out) - b.outPos
  1097. if outMax > s.lzma2.uncompressed {
  1098. outMax = s.lzma2.uncompressed
  1099. }
  1100. dictLimit(&s.dict, outMax)
  1101. if !lzma2LZMA(s, b) {
  1102. return xzDataError
  1103. }
  1104. s.lzma2.uncompressed -= dictFlush(&s.dict, b)
  1105. switch {
  1106. case s.lzma2.uncompressed == 0:
  1107. if s.lzma2.compressed > 0 || s.lzma.len > 0 ||
  1108. !rcIsFinished(&s.rc) {
  1109. return xzDataError
  1110. }
  1111. rcReset(&s.rc)
  1112. s.lzma2.sequence = seqControl
  1113. case b.outPos == len(b.out) ||
  1114. b.inPos == len(b.in) &&
  1115. len(s.temp.buf) < s.lzma2.compressed:
  1116. return xzOK
  1117. }
  1118. case seqCopy:
  1119. dictUncompressed(&s.dict, b, &s.lzma2.compressed)
  1120. if s.lzma2.compressed > 0 {
  1121. return xzOK
  1122. }
  1123. s.lzma2.sequence = seqControl
  1124. }
  1125. }
  1126. return xzOK
  1127. }
  1128. /*
  1129. * Allocate memory for LZMA2 decoder. xzDecLZMA2Reset must be used
  1130. * before calling xzDecLZMA2Run.
  1131. */
  1132. func xzDecLZMA2Create(dictMax uint32) *xzDecLZMA2 {
  1133. s := new(xzDecLZMA2)
  1134. s.dict.sizeMax = dictMax
  1135. return s
  1136. }
  1137. /*
  1138. * Decode the LZMA2 properties (one byte) and reset the decoder. Return
  1139. * xzOK on success, xzMemlimitError if the preallocated dictionary is not
  1140. * big enough, and xzOptionsError if props indicates something that this
  1141. * decoder doesn't support.
  1142. */
  1143. func xzDecLZMA2Reset(s *xzDecLZMA2, props byte) xzRet {
  1144. if props > 40 {
  1145. return xzOptionsError // Bigger than 4 GiB
  1146. }
  1147. if props == 40 {
  1148. s.dict.size = ^uint32(0)
  1149. } else {
  1150. s.dict.size = uint32(2 + props&1)
  1151. s.dict.size <<= props>>1 + 11
  1152. }
  1153. if s.dict.size > s.dict.sizeMax {
  1154. return xzMemlimitError
  1155. }
  1156. s.dict.end = s.dict.size
  1157. if len(s.dict.buf) < int(s.dict.size) {
  1158. s.dict.buf = make([]byte, s.dict.size)
  1159. }
  1160. s.lzma.len = 0
  1161. s.lzma2.sequence = seqControl
  1162. s.lzma2.compressed = 0
  1163. s.lzma2.uncompressed = 0
  1164. s.lzma2.needDictReset = true
  1165. s.temp.buf = nil
  1166. return xzOK
  1167. }