You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

arraycontainer.go 24KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980
  1. package roaring
  2. import (
  3. "fmt"
  4. )
  5. //go:generate msgp -unexported
  6. type arrayContainer struct {
  7. content []uint16
  8. }
  9. func (ac *arrayContainer) String() string {
  10. s := "{"
  11. for it := ac.getShortIterator(); it.hasNext(); {
  12. s += fmt.Sprintf("%v, ", it.next())
  13. }
  14. return s + "}"
  15. }
  16. func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) {
  17. for k := 0; k < len(ac.content); k++ {
  18. x[k+i] = uint32(ac.content[k]) | mask
  19. }
  20. }
  21. func (ac *arrayContainer) iterate(cb func(x uint16) bool) bool {
  22. iterator := shortIterator{ac.content, 0}
  23. for iterator.hasNext() {
  24. if !cb(iterator.next()) {
  25. return false
  26. }
  27. }
  28. return true
  29. }
  30. func (ac *arrayContainer) getShortIterator() shortPeekable {
  31. return &shortIterator{ac.content, 0}
  32. }
  33. func (ac *arrayContainer) getReverseIterator() shortIterable {
  34. return &reverseIterator{ac.content, len(ac.content) - 1}
  35. }
  36. func (ac *arrayContainer) getManyIterator() manyIterable {
  37. return &shortIterator{ac.content, 0}
  38. }
  39. func (ac *arrayContainer) minimum() uint16 {
  40. return ac.content[0] // assume not empty
  41. }
  42. func (ac *arrayContainer) maximum() uint16 {
  43. return ac.content[len(ac.content)-1] // assume not empty
  44. }
  45. func (ac *arrayContainer) getSizeInBytes() int {
  46. return ac.getCardinality() * 2
  47. }
  48. func (ac *arrayContainer) serializedSizeInBytes() int {
  49. return ac.getCardinality() * 2
  50. }
  51. func arrayContainerSizeInBytes(card int) int {
  52. return card * 2
  53. }
  54. // add the values in the range [firstOfRange,endx)
  55. func (ac *arrayContainer) iaddRange(firstOfRange, endx int) container {
  56. if firstOfRange >= endx {
  57. return ac
  58. }
  59. indexstart := binarySearch(ac.content, uint16(firstOfRange))
  60. if indexstart < 0 {
  61. indexstart = -indexstart - 1
  62. }
  63. indexend := binarySearch(ac.content, uint16(endx-1))
  64. if indexend < 0 {
  65. indexend = -indexend - 1
  66. } else {
  67. indexend++
  68. }
  69. rangelength := endx - firstOfRange
  70. newcardinality := indexstart + (ac.getCardinality() - indexend) + rangelength
  71. if newcardinality > arrayDefaultMaxSize {
  72. a := ac.toBitmapContainer()
  73. return a.iaddRange(firstOfRange, endx)
  74. }
  75. if cap(ac.content) < newcardinality {
  76. tmp := make([]uint16, newcardinality, newcardinality)
  77. copy(tmp[:indexstart], ac.content[:indexstart])
  78. copy(tmp[indexstart+rangelength:], ac.content[indexend:])
  79. ac.content = tmp
  80. } else {
  81. ac.content = ac.content[:newcardinality]
  82. copy(ac.content[indexstart+rangelength:], ac.content[indexend:])
  83. }
  84. for k := 0; k < rangelength; k++ {
  85. ac.content[k+indexstart] = uint16(firstOfRange + k)
  86. }
  87. return ac
  88. }
  89. // remove the values in the range [firstOfRange,endx)
  90. func (ac *arrayContainer) iremoveRange(firstOfRange, endx int) container {
  91. if firstOfRange >= endx {
  92. return ac
  93. }
  94. indexstart := binarySearch(ac.content, uint16(firstOfRange))
  95. if indexstart < 0 {
  96. indexstart = -indexstart - 1
  97. }
  98. indexend := binarySearch(ac.content, uint16(endx-1))
  99. if indexend < 0 {
  100. indexend = -indexend - 1
  101. } else {
  102. indexend++
  103. }
  104. rangelength := indexend - indexstart
  105. answer := ac
  106. copy(answer.content[indexstart:], ac.content[indexstart+rangelength:])
  107. answer.content = answer.content[:ac.getCardinality()-rangelength]
  108. return answer
  109. }
  110. // flip the values in the range [firstOfRange,endx)
  111. func (ac *arrayContainer) not(firstOfRange, endx int) container {
  112. if firstOfRange >= endx {
  113. return ac.clone()
  114. }
  115. return ac.notClose(firstOfRange, endx-1) // remove everything in [firstOfRange,endx-1]
  116. }
  117. // flip the values in the range [firstOfRange,lastOfRange]
  118. func (ac *arrayContainer) notClose(firstOfRange, lastOfRange int) container {
  119. if firstOfRange > lastOfRange { // unlike add and remove, not uses an inclusive range [firstOfRange,lastOfRange]
  120. return ac.clone()
  121. }
  122. // determine the span of array indices to be affected^M
  123. startIndex := binarySearch(ac.content, uint16(firstOfRange))
  124. if startIndex < 0 {
  125. startIndex = -startIndex - 1
  126. }
  127. lastIndex := binarySearch(ac.content, uint16(lastOfRange))
  128. if lastIndex < 0 {
  129. lastIndex = -lastIndex - 2
  130. }
  131. currentValuesInRange := lastIndex - startIndex + 1
  132. spanToBeFlipped := lastOfRange - firstOfRange + 1
  133. newValuesInRange := spanToBeFlipped - currentValuesInRange
  134. cardinalityChange := newValuesInRange - currentValuesInRange
  135. newCardinality := len(ac.content) + cardinalityChange
  136. if newCardinality > arrayDefaultMaxSize {
  137. return ac.toBitmapContainer().not(firstOfRange, lastOfRange+1)
  138. }
  139. answer := newArrayContainer()
  140. answer.content = make([]uint16, newCardinality, newCardinality) //a hack for sure
  141. copy(answer.content, ac.content[:startIndex])
  142. outPos := startIndex
  143. inPos := startIndex
  144. valInRange := firstOfRange
  145. for ; valInRange <= lastOfRange && inPos <= lastIndex; valInRange++ {
  146. if uint16(valInRange) != ac.content[inPos] {
  147. answer.content[outPos] = uint16(valInRange)
  148. outPos++
  149. } else {
  150. inPos++
  151. }
  152. }
  153. for ; valInRange <= lastOfRange; valInRange++ {
  154. answer.content[outPos] = uint16(valInRange)
  155. outPos++
  156. }
  157. for i := lastIndex + 1; i < len(ac.content); i++ {
  158. answer.content[outPos] = ac.content[i]
  159. outPos++
  160. }
  161. answer.content = answer.content[:newCardinality]
  162. return answer
  163. }
  164. func (ac *arrayContainer) equals(o container) bool {
  165. srb, ok := o.(*arrayContainer)
  166. if ok {
  167. // Check if the containers are the same object.
  168. if ac == srb {
  169. return true
  170. }
  171. if len(srb.content) != len(ac.content) {
  172. return false
  173. }
  174. for i, v := range ac.content {
  175. if v != srb.content[i] {
  176. return false
  177. }
  178. }
  179. return true
  180. }
  181. // use generic comparison
  182. bCard := o.getCardinality()
  183. aCard := ac.getCardinality()
  184. if bCard != aCard {
  185. return false
  186. }
  187. ait := ac.getShortIterator()
  188. bit := o.getShortIterator()
  189. for ait.hasNext() {
  190. if bit.next() != ait.next() {
  191. return false
  192. }
  193. }
  194. return true
  195. }
  196. func (ac *arrayContainer) toBitmapContainer() *bitmapContainer {
  197. bc := newBitmapContainer()
  198. bc.loadData(ac)
  199. return bc
  200. }
  201. func (ac *arrayContainer) iadd(x uint16) (wasNew bool) {
  202. // Special case adding to the end of the container.
  203. l := len(ac.content)
  204. if l > 0 && l < arrayDefaultMaxSize && ac.content[l-1] < x {
  205. ac.content = append(ac.content, x)
  206. return true
  207. }
  208. loc := binarySearch(ac.content, x)
  209. if loc < 0 {
  210. s := ac.content
  211. i := -loc - 1
  212. s = append(s, 0)
  213. copy(s[i+1:], s[i:])
  214. s[i] = x
  215. ac.content = s
  216. return true
  217. }
  218. return false
  219. }
  220. func (ac *arrayContainer) iaddReturnMinimized(x uint16) container {
  221. // Special case adding to the end of the container.
  222. l := len(ac.content)
  223. if l > 0 && l < arrayDefaultMaxSize && ac.content[l-1] < x {
  224. ac.content = append(ac.content, x)
  225. return ac
  226. }
  227. loc := binarySearch(ac.content, x)
  228. if loc < 0 {
  229. if len(ac.content) >= arrayDefaultMaxSize {
  230. a := ac.toBitmapContainer()
  231. a.iadd(x)
  232. return a
  233. }
  234. s := ac.content
  235. i := -loc - 1
  236. s = append(s, 0)
  237. copy(s[i+1:], s[i:])
  238. s[i] = x
  239. ac.content = s
  240. }
  241. return ac
  242. }
  243. // iremoveReturnMinimized is allowed to change the return type to minimize storage.
  244. func (ac *arrayContainer) iremoveReturnMinimized(x uint16) container {
  245. ac.iremove(x)
  246. return ac
  247. }
  248. func (ac *arrayContainer) iremove(x uint16) bool {
  249. loc := binarySearch(ac.content, x)
  250. if loc >= 0 {
  251. s := ac.content
  252. s = append(s[:loc], s[loc+1:]...)
  253. ac.content = s
  254. return true
  255. }
  256. return false
  257. }
  258. func (ac *arrayContainer) remove(x uint16) container {
  259. out := &arrayContainer{make([]uint16, len(ac.content))}
  260. copy(out.content, ac.content[:])
  261. loc := binarySearch(out.content, x)
  262. if loc >= 0 {
  263. s := out.content
  264. s = append(s[:loc], s[loc+1:]...)
  265. out.content = s
  266. }
  267. return out
  268. }
  269. func (ac *arrayContainer) or(a container) container {
  270. switch x := a.(type) {
  271. case *arrayContainer:
  272. return ac.orArray(x)
  273. case *bitmapContainer:
  274. return x.orArray(ac)
  275. case *runContainer16:
  276. if x.isFull() {
  277. return x.clone()
  278. }
  279. return x.orArray(ac)
  280. }
  281. panic("unsupported container type")
  282. }
  283. func (ac *arrayContainer) orCardinality(a container) int {
  284. switch x := a.(type) {
  285. case *arrayContainer:
  286. return ac.orArrayCardinality(x)
  287. case *bitmapContainer:
  288. return x.orArrayCardinality(ac)
  289. case *runContainer16:
  290. return x.orArrayCardinality(ac)
  291. }
  292. panic("unsupported container type")
  293. }
  294. func (ac *arrayContainer) ior(a container) container {
  295. switch x := a.(type) {
  296. case *arrayContainer:
  297. return ac.iorArray(x)
  298. case *bitmapContainer:
  299. return a.(*bitmapContainer).orArray(ac)
  300. //return ac.iorBitmap(x) // note: this does not make sense
  301. case *runContainer16:
  302. if x.isFull() {
  303. return x.clone()
  304. }
  305. return ac.iorRun16(x)
  306. }
  307. panic("unsupported container type")
  308. }
  309. func (ac *arrayContainer) iorArray(value2 *arrayContainer) container {
  310. value1 := ac
  311. len1 := value1.getCardinality()
  312. len2 := value2.getCardinality()
  313. maxPossibleCardinality := len1 + len2
  314. if maxPossibleCardinality > arrayDefaultMaxSize { // it could be a bitmap!
  315. bc := newBitmapContainer()
  316. for k := 0; k < len(value2.content); k++ {
  317. v := value2.content[k]
  318. i := uint(v) >> 6
  319. mask := uint64(1) << (v % 64)
  320. bc.bitmap[i] |= mask
  321. }
  322. for k := 0; k < len(ac.content); k++ {
  323. v := ac.content[k]
  324. i := uint(v) >> 6
  325. mask := uint64(1) << (v % 64)
  326. bc.bitmap[i] |= mask
  327. }
  328. bc.cardinality = int(popcntSlice(bc.bitmap))
  329. if bc.cardinality <= arrayDefaultMaxSize {
  330. return bc.toArrayContainer()
  331. }
  332. return bc
  333. }
  334. if maxPossibleCardinality > cap(value1.content) {
  335. newcontent := make([]uint16, 0, maxPossibleCardinality)
  336. copy(newcontent[len2:maxPossibleCardinality], ac.content[0:len1])
  337. ac.content = newcontent
  338. } else {
  339. copy(ac.content[len2:maxPossibleCardinality], ac.content[0:len1])
  340. }
  341. nl := union2by2(value1.content[len2:maxPossibleCardinality], value2.content, ac.content)
  342. ac.content = ac.content[:nl] // reslice to match actual used capacity
  343. return ac
  344. }
  345. // Note: such code does not make practical sense, except for lazy evaluations
  346. func (ac *arrayContainer) iorBitmap(bc2 *bitmapContainer) container {
  347. bc1 := ac.toBitmapContainer()
  348. bc1.iorBitmap(bc2)
  349. *ac = *newArrayContainerFromBitmap(bc1)
  350. return ac
  351. }
  352. func (ac *arrayContainer) iorRun16(rc *runContainer16) container {
  353. bc1 := ac.toBitmapContainer()
  354. bc2 := rc.toBitmapContainer()
  355. bc1.iorBitmap(bc2)
  356. *ac = *newArrayContainerFromBitmap(bc1)
  357. return ac
  358. }
  359. func (ac *arrayContainer) lazyIOR(a container) container {
  360. switch x := a.(type) {
  361. case *arrayContainer:
  362. return ac.lazyIorArray(x)
  363. case *bitmapContainer:
  364. return ac.lazyIorBitmap(x)
  365. case *runContainer16:
  366. if x.isFull() {
  367. return x.clone()
  368. }
  369. return ac.lazyIorRun16(x)
  370. }
  371. panic("unsupported container type")
  372. }
  373. func (ac *arrayContainer) lazyIorArray(ac2 *arrayContainer) container {
  374. // TODO actually make this lazy
  375. return ac.iorArray(ac2)
  376. }
  377. func (ac *arrayContainer) lazyIorBitmap(bc *bitmapContainer) container {
  378. // TODO actually make this lazy
  379. return ac.iorBitmap(bc)
  380. }
  381. func (ac *arrayContainer) lazyIorRun16(rc *runContainer16) container {
  382. // TODO actually make this lazy
  383. return ac.iorRun16(rc)
  384. }
  385. func (ac *arrayContainer) lazyOR(a container) container {
  386. switch x := a.(type) {
  387. case *arrayContainer:
  388. return ac.lazyorArray(x)
  389. case *bitmapContainer:
  390. return a.lazyOR(ac)
  391. case *runContainer16:
  392. if x.isFull() {
  393. return x.clone()
  394. }
  395. return x.orArray(ac)
  396. }
  397. panic("unsupported container type")
  398. }
  399. func (ac *arrayContainer) orArray(value2 *arrayContainer) container {
  400. value1 := ac
  401. maxPossibleCardinality := value1.getCardinality() + value2.getCardinality()
  402. if maxPossibleCardinality > arrayDefaultMaxSize { // it could be a bitmap!
  403. bc := newBitmapContainer()
  404. for k := 0; k < len(value2.content); k++ {
  405. v := value2.content[k]
  406. i := uint(v) >> 6
  407. mask := uint64(1) << (v % 64)
  408. bc.bitmap[i] |= mask
  409. }
  410. for k := 0; k < len(ac.content); k++ {
  411. v := ac.content[k]
  412. i := uint(v) >> 6
  413. mask := uint64(1) << (v % 64)
  414. bc.bitmap[i] |= mask
  415. }
  416. bc.cardinality = int(popcntSlice(bc.bitmap))
  417. if bc.cardinality <= arrayDefaultMaxSize {
  418. return bc.toArrayContainer()
  419. }
  420. return bc
  421. }
  422. answer := newArrayContainerCapacity(maxPossibleCardinality)
  423. nl := union2by2(value1.content, value2.content, answer.content)
  424. answer.content = answer.content[:nl] // reslice to match actual used capacity
  425. return answer
  426. }
  427. func (ac *arrayContainer) orArrayCardinality(value2 *arrayContainer) int {
  428. return union2by2Cardinality(ac.content, value2.content)
  429. }
  430. func (ac *arrayContainer) lazyorArray(value2 *arrayContainer) container {
  431. value1 := ac
  432. maxPossibleCardinality := value1.getCardinality() + value2.getCardinality()
  433. if maxPossibleCardinality > arrayLazyLowerBound { // it could be a bitmap!^M
  434. bc := newBitmapContainer()
  435. for k := 0; k < len(value2.content); k++ {
  436. v := value2.content[k]
  437. i := uint(v) >> 6
  438. mask := uint64(1) << (v % 64)
  439. bc.bitmap[i] |= mask
  440. }
  441. for k := 0; k < len(ac.content); k++ {
  442. v := ac.content[k]
  443. i := uint(v) >> 6
  444. mask := uint64(1) << (v % 64)
  445. bc.bitmap[i] |= mask
  446. }
  447. bc.cardinality = invalidCardinality
  448. return bc
  449. }
  450. answer := newArrayContainerCapacity(maxPossibleCardinality)
  451. nl := union2by2(value1.content, value2.content, answer.content)
  452. answer.content = answer.content[:nl] // reslice to match actual used capacity
  453. return answer
  454. }
  455. func (ac *arrayContainer) and(a container) container {
  456. switch x := a.(type) {
  457. case *arrayContainer:
  458. return ac.andArray(x)
  459. case *bitmapContainer:
  460. return x.and(ac)
  461. case *runContainer16:
  462. if x.isFull() {
  463. return ac.clone()
  464. }
  465. return x.andArray(ac)
  466. }
  467. panic("unsupported container type")
  468. }
  469. func (ac *arrayContainer) andCardinality(a container) int {
  470. switch x := a.(type) {
  471. case *arrayContainer:
  472. return ac.andArrayCardinality(x)
  473. case *bitmapContainer:
  474. return x.andCardinality(ac)
  475. case *runContainer16:
  476. return x.andArrayCardinality(ac)
  477. }
  478. panic("unsupported container type")
  479. }
  480. func (ac *arrayContainer) intersects(a container) bool {
  481. switch x := a.(type) {
  482. case *arrayContainer:
  483. return ac.intersectsArray(x)
  484. case *bitmapContainer:
  485. return x.intersects(ac)
  486. case *runContainer16:
  487. return x.intersects(ac)
  488. }
  489. panic("unsupported container type")
  490. }
  491. func (ac *arrayContainer) iand(a container) container {
  492. switch x := a.(type) {
  493. case *arrayContainer:
  494. return ac.iandArray(x)
  495. case *bitmapContainer:
  496. return ac.iandBitmap(x)
  497. case *runContainer16:
  498. if x.isFull() {
  499. return ac
  500. }
  501. return x.andArray(ac)
  502. }
  503. panic("unsupported container type")
  504. }
  505. func (ac *arrayContainer) iandBitmap(bc *bitmapContainer) container {
  506. pos := 0
  507. c := ac.getCardinality()
  508. for k := 0; k < c; k++ {
  509. // branchless
  510. v := ac.content[k]
  511. ac.content[pos] = v
  512. pos += int(bc.bitValue(v))
  513. }
  514. ac.content = ac.content[:pos]
  515. return ac
  516. }
  517. func (ac *arrayContainer) xor(a container) container {
  518. switch x := a.(type) {
  519. case *arrayContainer:
  520. return ac.xorArray(x)
  521. case *bitmapContainer:
  522. return a.xor(ac)
  523. case *runContainer16:
  524. return x.xorArray(ac)
  525. }
  526. panic("unsupported container type")
  527. }
  528. func (ac *arrayContainer) xorArray(value2 *arrayContainer) container {
  529. value1 := ac
  530. totalCardinality := value1.getCardinality() + value2.getCardinality()
  531. if totalCardinality > arrayDefaultMaxSize { // it could be a bitmap!
  532. bc := newBitmapContainer()
  533. for k := 0; k < len(value2.content); k++ {
  534. v := value2.content[k]
  535. i := uint(v) >> 6
  536. bc.bitmap[i] ^= (uint64(1) << (v % 64))
  537. }
  538. for k := 0; k < len(ac.content); k++ {
  539. v := ac.content[k]
  540. i := uint(v) >> 6
  541. bc.bitmap[i] ^= (uint64(1) << (v % 64))
  542. }
  543. bc.computeCardinality()
  544. if bc.cardinality <= arrayDefaultMaxSize {
  545. return bc.toArrayContainer()
  546. }
  547. return bc
  548. }
  549. desiredCapacity := totalCardinality
  550. answer := newArrayContainerCapacity(desiredCapacity)
  551. length := exclusiveUnion2by2(value1.content, value2.content, answer.content)
  552. answer.content = answer.content[:length]
  553. return answer
  554. }
  555. func (ac *arrayContainer) andNot(a container) container {
  556. switch x := a.(type) {
  557. case *arrayContainer:
  558. return ac.andNotArray(x)
  559. case *bitmapContainer:
  560. return ac.andNotBitmap(x)
  561. case *runContainer16:
  562. return ac.andNotRun16(x)
  563. }
  564. panic("unsupported container type")
  565. }
  566. func (ac *arrayContainer) andNotRun16(rc *runContainer16) container {
  567. acb := ac.toBitmapContainer()
  568. rcb := rc.toBitmapContainer()
  569. return acb.andNotBitmap(rcb)
  570. }
  571. func (ac *arrayContainer) iandNot(a container) container {
  572. switch x := a.(type) {
  573. case *arrayContainer:
  574. return ac.iandNotArray(x)
  575. case *bitmapContainer:
  576. return ac.iandNotBitmap(x)
  577. case *runContainer16:
  578. return ac.iandNotRun16(x)
  579. }
  580. panic("unsupported container type")
  581. }
  582. func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container {
  583. rcb := rc.toBitmapContainer()
  584. acb := ac.toBitmapContainer()
  585. acb.iandNotBitmapSurely(rcb)
  586. *ac = *(acb.toArrayContainer())
  587. return ac
  588. }
  589. func (ac *arrayContainer) andNotArray(value2 *arrayContainer) container {
  590. value1 := ac
  591. desiredcapacity := value1.getCardinality()
  592. answer := newArrayContainerCapacity(desiredcapacity)
  593. length := difference(value1.content, value2.content, answer.content)
  594. answer.content = answer.content[:length]
  595. return answer
  596. }
  597. func (ac *arrayContainer) iandNotArray(value2 *arrayContainer) container {
  598. length := difference(ac.content, value2.content, ac.content)
  599. ac.content = ac.content[:length]
  600. return ac
  601. }
  602. func (ac *arrayContainer) andNotBitmap(value2 *bitmapContainer) container {
  603. desiredcapacity := ac.getCardinality()
  604. answer := newArrayContainerCapacity(desiredcapacity)
  605. answer.content = answer.content[:desiredcapacity]
  606. pos := 0
  607. for _, v := range ac.content {
  608. answer.content[pos] = v
  609. pos += 1 - int(value2.bitValue(v))
  610. }
  611. answer.content = answer.content[:pos]
  612. return answer
  613. }
  614. func (ac *arrayContainer) andBitmap(value2 *bitmapContainer) container {
  615. desiredcapacity := ac.getCardinality()
  616. answer := newArrayContainerCapacity(desiredcapacity)
  617. answer.content = answer.content[:desiredcapacity]
  618. pos := 0
  619. for _, v := range ac.content {
  620. answer.content[pos] = v
  621. pos += int(value2.bitValue(v))
  622. }
  623. answer.content = answer.content[:pos]
  624. return answer
  625. }
  626. func (ac *arrayContainer) iandNotBitmap(value2 *bitmapContainer) container {
  627. pos := 0
  628. for _, v := range ac.content {
  629. ac.content[pos] = v
  630. pos += 1 - int(value2.bitValue(v))
  631. }
  632. ac.content = ac.content[:pos]
  633. return ac
  634. }
  635. func copyOf(array []uint16, size int) []uint16 {
  636. result := make([]uint16, size)
  637. for i, x := range array {
  638. if i == size {
  639. break
  640. }
  641. result[i] = x
  642. }
  643. return result
  644. }
  645. // flip the values in the range [firstOfRange,endx)
  646. func (ac *arrayContainer) inot(firstOfRange, endx int) container {
  647. if firstOfRange >= endx {
  648. return ac
  649. }
  650. return ac.inotClose(firstOfRange, endx-1) // remove everything in [firstOfRange,endx-1]
  651. }
  652. // flip the values in the range [firstOfRange,lastOfRange]
  653. func (ac *arrayContainer) inotClose(firstOfRange, lastOfRange int) container {
  654. if firstOfRange > lastOfRange { // unlike add and remove, not uses an inclusive range [firstOfRange,lastOfRange]
  655. return ac
  656. }
  657. // determine the span of array indices to be affected
  658. startIndex := binarySearch(ac.content, uint16(firstOfRange))
  659. if startIndex < 0 {
  660. startIndex = -startIndex - 1
  661. }
  662. lastIndex := binarySearch(ac.content, uint16(lastOfRange))
  663. if lastIndex < 0 {
  664. lastIndex = -lastIndex - 1 - 1
  665. }
  666. currentValuesInRange := lastIndex - startIndex + 1
  667. spanToBeFlipped := lastOfRange - firstOfRange + 1
  668. newValuesInRange := spanToBeFlipped - currentValuesInRange
  669. buffer := make([]uint16, newValuesInRange)
  670. cardinalityChange := newValuesInRange - currentValuesInRange
  671. newCardinality := len(ac.content) + cardinalityChange
  672. if cardinalityChange > 0 {
  673. if newCardinality > len(ac.content) {
  674. if newCardinality > arrayDefaultMaxSize {
  675. bcRet := ac.toBitmapContainer()
  676. bcRet.inot(firstOfRange, lastOfRange+1)
  677. *ac = *bcRet.toArrayContainer()
  678. return bcRet
  679. }
  680. ac.content = copyOf(ac.content, newCardinality)
  681. }
  682. base := lastIndex + 1
  683. copy(ac.content[lastIndex+1+cardinalityChange:], ac.content[base:base+len(ac.content)-1-lastIndex])
  684. ac.negateRange(buffer, startIndex, lastIndex, firstOfRange, lastOfRange+1)
  685. } else { // no expansion needed
  686. ac.negateRange(buffer, startIndex, lastIndex, firstOfRange, lastOfRange+1)
  687. if cardinalityChange < 0 {
  688. for i := startIndex + newValuesInRange; i < newCardinality; i++ {
  689. ac.content[i] = ac.content[i-cardinalityChange]
  690. }
  691. }
  692. }
  693. ac.content = ac.content[:newCardinality]
  694. return ac
  695. }
  696. func (ac *arrayContainer) negateRange(buffer []uint16, startIndex, lastIndex, startRange, lastRange int) {
  697. // compute the negation into buffer
  698. outPos := 0
  699. inPos := startIndex // value here always >= valInRange,
  700. // until it is exhausted
  701. // n.b., we can start initially exhausted.
  702. valInRange := startRange
  703. for ; valInRange < lastRange && inPos <= lastIndex; valInRange++ {
  704. if uint16(valInRange) != ac.content[inPos] {
  705. buffer[outPos] = uint16(valInRange)
  706. outPos++
  707. } else {
  708. inPos++
  709. }
  710. }
  711. // if there are extra items (greater than the biggest
  712. // pre-existing one in range), buffer them
  713. for ; valInRange < lastRange; valInRange++ {
  714. buffer[outPos] = uint16(valInRange)
  715. outPos++
  716. }
  717. if outPos != len(buffer) {
  718. panic("negateRange: internal bug")
  719. }
  720. for i, item := range buffer {
  721. ac.content[i+startIndex] = item
  722. }
  723. }
  724. func (ac *arrayContainer) isFull() bool {
  725. return false
  726. }
  727. func (ac *arrayContainer) andArray(value2 *arrayContainer) container {
  728. desiredcapacity := minOfInt(ac.getCardinality(), value2.getCardinality())
  729. answer := newArrayContainerCapacity(desiredcapacity)
  730. length := intersection2by2(
  731. ac.content,
  732. value2.content,
  733. answer.content)
  734. answer.content = answer.content[:length]
  735. return answer
  736. }
  737. func (ac *arrayContainer) andArrayCardinality(value2 *arrayContainer) int {
  738. return intersection2by2Cardinality(
  739. ac.content,
  740. value2.content)
  741. }
  742. func (ac *arrayContainer) intersectsArray(value2 *arrayContainer) bool {
  743. return intersects2by2(
  744. ac.content,
  745. value2.content)
  746. }
  747. func (ac *arrayContainer) iandArray(value2 *arrayContainer) container {
  748. length := intersection2by2(
  749. ac.content,
  750. value2.content,
  751. ac.content)
  752. ac.content = ac.content[:length]
  753. return ac
  754. }
  755. func (ac *arrayContainer) getCardinality() int {
  756. return len(ac.content)
  757. }
  758. func (ac *arrayContainer) rank(x uint16) int {
  759. answer := binarySearch(ac.content, x)
  760. if answer >= 0 {
  761. return answer + 1
  762. }
  763. return -answer - 1
  764. }
  765. func (ac *arrayContainer) selectInt(x uint16) int {
  766. return int(ac.content[x])
  767. }
  768. func (ac *arrayContainer) clone() container {
  769. ptr := arrayContainer{make([]uint16, len(ac.content))}
  770. copy(ptr.content, ac.content[:])
  771. return &ptr
  772. }
  773. func (ac *arrayContainer) contains(x uint16) bool {
  774. return binarySearch(ac.content, x) >= 0
  775. }
  776. func (ac *arrayContainer) loadData(bitmapContainer *bitmapContainer) {
  777. ac.content = make([]uint16, bitmapContainer.cardinality, bitmapContainer.cardinality)
  778. bitmapContainer.fillArray(ac.content)
  779. }
  780. func newArrayContainer() *arrayContainer {
  781. p := new(arrayContainer)
  782. return p
  783. }
  784. func newArrayContainerFromBitmap(bc *bitmapContainer) *arrayContainer {
  785. ac := &arrayContainer{}
  786. ac.loadData(bc)
  787. return ac
  788. }
  789. func newArrayContainerCapacity(size int) *arrayContainer {
  790. p := new(arrayContainer)
  791. p.content = make([]uint16, 0, size)
  792. return p
  793. }
  794. func newArrayContainerSize(size int) *arrayContainer {
  795. p := new(arrayContainer)
  796. p.content = make([]uint16, size, size)
  797. return p
  798. }
  799. func newArrayContainerRange(firstOfRun, lastOfRun int) *arrayContainer {
  800. valuesInRange := lastOfRun - firstOfRun + 1
  801. this := newArrayContainerCapacity(valuesInRange)
  802. for i := 0; i < valuesInRange; i++ {
  803. this.content = append(this.content, uint16(firstOfRun+i))
  804. }
  805. return this
  806. }
  807. func (ac *arrayContainer) numberOfRuns() (nr int) {
  808. n := len(ac.content)
  809. var runlen uint16
  810. var cur, prev uint16
  811. switch n {
  812. case 0:
  813. return 0
  814. case 1:
  815. return 1
  816. default:
  817. for i := 1; i < n; i++ {
  818. prev = ac.content[i-1]
  819. cur = ac.content[i]
  820. if cur == prev+1 {
  821. runlen++
  822. } else {
  823. if cur < prev {
  824. panic("then fundamental arrayContainer assumption of sorted ac.content was broken")
  825. }
  826. if cur == prev {
  827. panic("then fundamental arrayContainer assumption of deduplicated content was broken")
  828. } else {
  829. nr++
  830. runlen = 0
  831. }
  832. }
  833. }
  834. nr++
  835. }
  836. return
  837. }
  838. // convert to run or array *if needed*
  839. func (ac *arrayContainer) toEfficientContainer() container {
  840. numRuns := ac.numberOfRuns()
  841. sizeAsRunContainer := runContainer16SerializedSizeInBytes(numRuns)
  842. sizeAsBitmapContainer := bitmapContainerSizeInBytes()
  843. card := ac.getCardinality()
  844. sizeAsArrayContainer := arrayContainerSizeInBytes(card)
  845. if sizeAsRunContainer <= minOfInt(sizeAsBitmapContainer, sizeAsArrayContainer) {
  846. return newRunContainer16FromArray(ac)
  847. }
  848. if card <= arrayDefaultMaxSize {
  849. return ac
  850. }
  851. return ac.toBitmapContainer()
  852. }
  853. func (ac *arrayContainer) containerType() contype {
  854. return arrayContype
  855. }
  856. func (ac *arrayContainer) addOffset(x uint16) []container {
  857. low := &arrayContainer{}
  858. high := &arrayContainer{}
  859. for _, val := range ac.content {
  860. y := uint32(val) + uint32(x)
  861. if highbits(y) > 0 {
  862. high.content = append(high.content, lowbits(y))
  863. } else {
  864. low.content = append(low.content, lowbits(y))
  865. }
  866. }
  867. return []container{low, high}
  868. }