You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

roaring.go 44KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494
  1. // Package roaring is an implementation of Roaring Bitmaps in Go.
  2. // They provide fast compressed bitmap data structures (also called bitset).
  3. // They are ideally suited to represent sets of integers over
  4. // relatively small ranges.
  5. // See http://roaringbitmap.org for details.
  6. package roaring
  7. import (
  8. "bytes"
  9. "encoding/base64"
  10. "fmt"
  11. "io"
  12. "strconv"
  13. "sync"
  14. )
  15. // Bitmap represents a compressed bitmap where you can add integers.
  16. type Bitmap struct {
  17. highlowcontainer roaringArray
  18. }
  19. // ToBase64 serializes a bitmap as Base64
  20. func (rb *Bitmap) ToBase64() (string, error) {
  21. buf := new(bytes.Buffer)
  22. _, err := rb.WriteTo(buf)
  23. return base64.StdEncoding.EncodeToString(buf.Bytes()), err
  24. }
  25. // FromBase64 deserializes a bitmap from Base64
  26. func (rb *Bitmap) FromBase64(str string) (int64, error) {
  27. data, err := base64.StdEncoding.DecodeString(str)
  28. if err != nil {
  29. return 0, err
  30. }
  31. buf := bytes.NewBuffer(data)
  32. return rb.ReadFrom(buf)
  33. }
  34. // WriteTo writes a serialized version of this bitmap to stream.
  35. // The format is compatible with other RoaringBitmap
  36. // implementations (Java, C) and is documented here:
  37. // https://github.com/RoaringBitmap/RoaringFormatSpec
  38. func (rb *Bitmap) WriteTo(stream io.Writer) (int64, error) {
  39. return rb.highlowcontainer.writeTo(stream)
  40. }
  41. // ToBytes returns an array of bytes corresponding to what is written
  42. // when calling WriteTo
  43. func (rb *Bitmap) ToBytes() ([]byte, error) {
  44. return rb.highlowcontainer.toBytes()
  45. }
  46. // Deprecated: WriteToMsgpack writes a msgpack2/snappy-streaming compressed serialized
  47. // version of this bitmap to stream. The format is not
  48. // compatible with the WriteTo() format, and is
  49. // experimental: it may produce smaller on disk
  50. // footprint and/or be faster to read, depending
  51. // on your content. Currently only the Go roaring
  52. // implementation supports this format.
  53. func (rb *Bitmap) WriteToMsgpack(stream io.Writer) (int64, error) {
  54. return 0, rb.highlowcontainer.writeToMsgpack(stream)
  55. }
  56. // ReadFrom reads a serialized version of this bitmap from stream.
  57. // The format is compatible with other RoaringBitmap
  58. // implementations (Java, C) and is documented here:
  59. // https://github.com/RoaringBitmap/RoaringFormatSpec
  60. func (rb *Bitmap) ReadFrom(reader io.Reader) (p int64, err error) {
  61. stream := byteInputAdapterPool.Get().(*byteInputAdapter)
  62. stream.reset(reader)
  63. p, err = rb.highlowcontainer.readFrom(stream)
  64. byteInputAdapterPool.Put(stream)
  65. return
  66. }
  67. // FromBuffer creates a bitmap from its serialized version stored in buffer
  68. //
  69. // The format specification is available here:
  70. // https://github.com/RoaringBitmap/RoaringFormatSpec
  71. //
  72. // The provided byte array (buf) is expected to be a constant.
  73. // The function makes the best effort attempt not to copy data.
  74. // You should take care not to modify buff as it will
  75. // likely result in unexpected program behavior.
  76. //
  77. // Resulting bitmaps are effectively immutable in the following sense:
  78. // a copy-on-write marker is used so that when you modify the resulting
  79. // bitmap, copies of selected data (containers) are made.
  80. // You should *not* change the copy-on-write status of the resulting
  81. // bitmaps (SetCopyOnWrite).
  82. //
  83. // If buf becomes unavailable, then a bitmap created with
  84. // FromBuffer would be effectively broken. Furthermore, any
  85. // bitmap derived from this bitmap (e.g., via Or, And) might
  86. // also be broken. Thus, before making buf unavailable, you should
  87. // call CloneCopyOnWriteContainers on all such bitmaps.
  88. //
  89. func (rb *Bitmap) FromBuffer(buf []byte) (p int64, err error) {
  90. stream := byteBufferPool.Get().(*byteBuffer)
  91. stream.reset(buf)
  92. p, err = rb.highlowcontainer.readFrom(stream)
  93. byteBufferPool.Put(stream)
  94. return
  95. }
  96. var (
  97. byteBufferPool = sync.Pool{
  98. New: func() interface{} {
  99. return &byteBuffer{}
  100. },
  101. }
  102. byteInputAdapterPool = sync.Pool{
  103. New: func() interface{} {
  104. return &byteInputAdapter{}
  105. },
  106. }
  107. )
  108. // RunOptimize attempts to further compress the runs of consecutive values found in the bitmap
  109. func (rb *Bitmap) RunOptimize() {
  110. rb.highlowcontainer.runOptimize()
  111. }
  112. // HasRunCompression returns true if the bitmap benefits from run compression
  113. func (rb *Bitmap) HasRunCompression() bool {
  114. return rb.highlowcontainer.hasRunCompression()
  115. }
  116. // Deprecated: ReadFromMsgpack reads a msgpack2/snappy-streaming serialized
  117. // version of this bitmap from stream. The format is
  118. // expected is that written by the WriteToMsgpack()
  119. // call; see additional notes there.
  120. func (rb *Bitmap) ReadFromMsgpack(stream io.Reader) (int64, error) {
  121. return 0, rb.highlowcontainer.readFromMsgpack(stream)
  122. }
  123. // MarshalBinary implements the encoding.BinaryMarshaler interface for the bitmap
  124. // (same as ToBytes)
  125. func (rb *Bitmap) MarshalBinary() ([]byte, error) {
  126. return rb.ToBytes()
  127. }
  128. // UnmarshalBinary implements the encoding.BinaryUnmarshaler interface for the bitmap
  129. func (rb *Bitmap) UnmarshalBinary(data []byte) error {
  130. r := bytes.NewReader(data)
  131. _, err := rb.ReadFrom(r)
  132. return err
  133. }
  134. // NewBitmap creates a new empty Bitmap (see also New)
  135. func NewBitmap() *Bitmap {
  136. return &Bitmap{}
  137. }
  138. // New creates a new empty Bitmap (same as NewBitmap)
  139. func New() *Bitmap {
  140. return &Bitmap{}
  141. }
  142. // Clear resets the Bitmap to be logically empty, but may retain
  143. // some memory allocations that may speed up future operations
  144. func (rb *Bitmap) Clear() {
  145. rb.highlowcontainer.clear()
  146. }
  147. // ToArray creates a new slice containing all of the integers stored in the Bitmap in sorted order
  148. func (rb *Bitmap) ToArray() []uint32 {
  149. array := make([]uint32, rb.GetCardinality())
  150. pos := 0
  151. pos2 := 0
  152. for pos < rb.highlowcontainer.size() {
  153. hs := uint32(rb.highlowcontainer.getKeyAtIndex(pos)) << 16
  154. c := rb.highlowcontainer.getContainerAtIndex(pos)
  155. pos++
  156. c.fillLeastSignificant16bits(array, pos2, hs)
  157. pos2 += c.getCardinality()
  158. }
  159. return array
  160. }
  161. // GetSizeInBytes estimates the memory usage of the Bitmap. Note that this
  162. // might differ slightly from the amount of bytes required for persistent storage
  163. func (rb *Bitmap) GetSizeInBytes() uint64 {
  164. size := uint64(8)
  165. for _, c := range rb.highlowcontainer.containers {
  166. size += uint64(2) + uint64(c.getSizeInBytes())
  167. }
  168. return size
  169. }
  170. // GetSerializedSizeInBytes computes the serialized size in bytes
  171. // of the Bitmap. It should correspond to the
  172. // number of bytes written when invoking WriteTo. You can expect
  173. // that this function is much cheaper computationally than WriteTo.
  174. func (rb *Bitmap) GetSerializedSizeInBytes() uint64 {
  175. return rb.highlowcontainer.serializedSizeInBytes()
  176. }
  177. // BoundSerializedSizeInBytes returns an upper bound on the serialized size in bytes
  178. // assuming that one wants to store "cardinality" integers in [0, universe_size)
  179. func BoundSerializedSizeInBytes(cardinality uint64, universeSize uint64) uint64 {
  180. contnbr := (universeSize + uint64(65535)) / uint64(65536)
  181. if contnbr > cardinality {
  182. contnbr = cardinality
  183. // we can't have more containers than we have values
  184. }
  185. headermax := 8*contnbr + 4
  186. if 4 > (contnbr+7)/8 {
  187. headermax += 4
  188. } else {
  189. headermax += (contnbr + 7) / 8
  190. }
  191. valsarray := uint64(arrayContainerSizeInBytes(int(cardinality)))
  192. valsbitmap := contnbr * uint64(bitmapContainerSizeInBytes())
  193. valsbest := valsarray
  194. if valsbest > valsbitmap {
  195. valsbest = valsbitmap
  196. }
  197. return valsbest + headermax
  198. }
  199. // IntIterable allows you to iterate over the values in a Bitmap
  200. type IntIterable interface {
  201. HasNext() bool
  202. Next() uint32
  203. }
  204. // IntPeekable allows you to look at the next value without advancing and
  205. // advance as long as the next value is smaller than minval
  206. type IntPeekable interface {
  207. IntIterable
  208. // PeekNext peeks the next value without advancing the iterator
  209. PeekNext() uint32
  210. // AdvanceIfNeeded advances as long as the next value is smaller than minval
  211. AdvanceIfNeeded(minval uint32)
  212. }
  213. type intIterator struct {
  214. pos int
  215. hs uint32
  216. iter shortPeekable
  217. highlowcontainer *roaringArray
  218. }
  219. // HasNext returns true if there are more integers to iterate over
  220. func (ii *intIterator) HasNext() bool {
  221. return ii.pos < ii.highlowcontainer.size()
  222. }
  223. func (ii *intIterator) init() {
  224. if ii.highlowcontainer.size() > ii.pos {
  225. ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getShortIterator()
  226. ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16
  227. }
  228. }
  229. // Next returns the next integer
  230. func (ii *intIterator) Next() uint32 {
  231. x := uint32(ii.iter.next()) | ii.hs
  232. if !ii.iter.hasNext() {
  233. ii.pos = ii.pos + 1
  234. ii.init()
  235. }
  236. return x
  237. }
  238. // PeekNext peeks the next value without advancing the iterator
  239. func (ii *intIterator) PeekNext() uint32 {
  240. return uint32(ii.iter.peekNext()&maxLowBit) | ii.hs
  241. }
  242. // AdvanceIfNeeded advances as long as the next value is smaller than minval
  243. func (ii *intIterator) AdvanceIfNeeded(minval uint32) {
  244. to := minval >> 16
  245. for ii.HasNext() && (ii.hs>>16) < to {
  246. ii.pos++
  247. ii.init()
  248. }
  249. if ii.HasNext() && (ii.hs>>16) == to {
  250. ii.iter.advanceIfNeeded(lowbits(minval))
  251. if !ii.iter.hasNext() {
  252. ii.pos++
  253. ii.init()
  254. }
  255. }
  256. }
  257. func newIntIterator(a *Bitmap) *intIterator {
  258. p := new(intIterator)
  259. p.pos = 0
  260. p.highlowcontainer = &a.highlowcontainer
  261. p.init()
  262. return p
  263. }
  264. type intReverseIterator struct {
  265. pos int
  266. hs uint32
  267. iter shortIterable
  268. highlowcontainer *roaringArray
  269. }
  270. // HasNext returns true if there are more integers to iterate over
  271. func (ii *intReverseIterator) HasNext() bool {
  272. return ii.pos >= 0
  273. }
  274. func (ii *intReverseIterator) init() {
  275. if ii.pos >= 0 {
  276. ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getReverseIterator()
  277. ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16
  278. } else {
  279. ii.iter = nil
  280. }
  281. }
  282. // Next returns the next integer
  283. func (ii *intReverseIterator) Next() uint32 {
  284. x := uint32(ii.iter.next()) | ii.hs
  285. if !ii.iter.hasNext() {
  286. ii.pos = ii.pos - 1
  287. ii.init()
  288. }
  289. return x
  290. }
  291. func newIntReverseIterator(a *Bitmap) *intReverseIterator {
  292. p := new(intReverseIterator)
  293. p.highlowcontainer = &a.highlowcontainer
  294. p.pos = a.highlowcontainer.size() - 1
  295. p.init()
  296. return p
  297. }
  298. // ManyIntIterable allows you to iterate over the values in a Bitmap
  299. type ManyIntIterable interface {
  300. // pass in a buffer to fill up with values, returns how many values were returned
  301. NextMany([]uint32) int
  302. }
  303. type manyIntIterator struct {
  304. pos int
  305. hs uint32
  306. iter manyIterable
  307. highlowcontainer *roaringArray
  308. }
  309. func (ii *manyIntIterator) init() {
  310. if ii.highlowcontainer.size() > ii.pos {
  311. ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getManyIterator()
  312. ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16
  313. } else {
  314. ii.iter = nil
  315. }
  316. }
  317. func (ii *manyIntIterator) NextMany(buf []uint32) int {
  318. n := 0
  319. for n < len(buf) {
  320. if ii.iter == nil {
  321. break
  322. }
  323. moreN := ii.iter.nextMany(ii.hs, buf[n:])
  324. n += moreN
  325. if moreN == 0 {
  326. ii.pos = ii.pos + 1
  327. ii.init()
  328. }
  329. }
  330. return n
  331. }
  332. func newManyIntIterator(a *Bitmap) *manyIntIterator {
  333. p := new(manyIntIterator)
  334. p.pos = 0
  335. p.highlowcontainer = &a.highlowcontainer
  336. p.init()
  337. return p
  338. }
  339. // String creates a string representation of the Bitmap
  340. func (rb *Bitmap) String() string {
  341. // inspired by https://github.com/fzandona/goroar/
  342. var buffer bytes.Buffer
  343. start := []byte("{")
  344. buffer.Write(start)
  345. i := rb.Iterator()
  346. counter := 0
  347. if i.HasNext() {
  348. counter = counter + 1
  349. buffer.WriteString(strconv.FormatInt(int64(i.Next()), 10))
  350. }
  351. for i.HasNext() {
  352. buffer.WriteString(",")
  353. counter = counter + 1
  354. // to avoid exhausting the memory
  355. if counter > 0x40000 {
  356. buffer.WriteString("...")
  357. break
  358. }
  359. buffer.WriteString(strconv.FormatInt(int64(i.Next()), 10))
  360. }
  361. buffer.WriteString("}")
  362. return buffer.String()
  363. }
  364. // Iterator creates a new IntPeekable to iterate over the integers contained in the bitmap, in sorted order;
  365. // the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove).
  366. func (rb *Bitmap) Iterator() IntPeekable {
  367. return newIntIterator(rb)
  368. }
  369. // ReverseIterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order;
  370. // the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove).
  371. func (rb *Bitmap) ReverseIterator() IntIterable {
  372. return newIntReverseIterator(rb)
  373. }
  374. // ManyIterator creates a new ManyIntIterable to iterate over the integers contained in the bitmap, in sorted order;
  375. // the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove).
  376. func (rb *Bitmap) ManyIterator() ManyIntIterable {
  377. return newManyIntIterator(rb)
  378. }
  379. // Clone creates a copy of the Bitmap
  380. func (rb *Bitmap) Clone() *Bitmap {
  381. ptr := new(Bitmap)
  382. ptr.highlowcontainer = *rb.highlowcontainer.clone()
  383. return ptr
  384. }
  385. // Minimum get the smallest value stored in this roaring bitmap, assumes that it is not empty
  386. func (rb *Bitmap) Minimum() uint32 {
  387. return uint32(rb.highlowcontainer.containers[0].minimum()) | (uint32(rb.highlowcontainer.keys[0]) << 16)
  388. }
  389. // Maximum get the largest value stored in this roaring bitmap, assumes that it is not empty
  390. func (rb *Bitmap) Maximum() uint32 {
  391. lastindex := len(rb.highlowcontainer.containers) - 1
  392. return uint32(rb.highlowcontainer.containers[lastindex].maximum()) | (uint32(rb.highlowcontainer.keys[lastindex]) << 16)
  393. }
  394. // Contains returns true if the integer is contained in the bitmap
  395. func (rb *Bitmap) Contains(x uint32) bool {
  396. hb := highbits(x)
  397. c := rb.highlowcontainer.getContainer(hb)
  398. return c != nil && c.contains(lowbits(x))
  399. }
  400. // ContainsInt returns true if the integer is contained in the bitmap (this is a convenience method, the parameter is casted to uint32 and Contains is called)
  401. func (rb *Bitmap) ContainsInt(x int) bool {
  402. return rb.Contains(uint32(x))
  403. }
  404. // Equals returns true if the two bitmaps contain the same integers
  405. func (rb *Bitmap) Equals(o interface{}) bool {
  406. srb, ok := o.(*Bitmap)
  407. if ok {
  408. return srb.highlowcontainer.equals(rb.highlowcontainer)
  409. }
  410. return false
  411. }
  412. // AddOffset adds the value 'offset' to each and every value in a bitmap, generating a new bitmap in the process
  413. func AddOffset(x *Bitmap, offset uint32) (answer *Bitmap) {
  414. containerOffset := highbits(offset)
  415. inOffset := lowbits(offset)
  416. if inOffset == 0 {
  417. answer = x.Clone()
  418. for pos := 0; pos < answer.highlowcontainer.size(); pos++ {
  419. key := answer.highlowcontainer.getKeyAtIndex(pos)
  420. key += containerOffset
  421. answer.highlowcontainer.keys[pos] = key
  422. }
  423. } else {
  424. answer = New()
  425. for pos := 0; pos < x.highlowcontainer.size(); pos++ {
  426. key := x.highlowcontainer.getKeyAtIndex(pos)
  427. key += containerOffset
  428. c := x.highlowcontainer.getContainerAtIndex(pos)
  429. offsetted := c.addOffset(inOffset)
  430. if offsetted[0].getCardinality() > 0 {
  431. curSize := answer.highlowcontainer.size()
  432. lastkey := uint16(0)
  433. if curSize > 0 {
  434. lastkey = answer.highlowcontainer.getKeyAtIndex(curSize - 1)
  435. }
  436. if curSize > 0 && lastkey == key {
  437. prev := answer.highlowcontainer.getContainerAtIndex(curSize - 1)
  438. orrseult := prev.ior(offsetted[0])
  439. answer.highlowcontainer.setContainerAtIndex(curSize-1, orrseult)
  440. } else {
  441. answer.highlowcontainer.appendContainer(key, offsetted[0], false)
  442. }
  443. }
  444. if offsetted[1].getCardinality() > 0 {
  445. answer.highlowcontainer.appendContainer(key+1, offsetted[1], false)
  446. }
  447. }
  448. }
  449. return answer
  450. }
  451. // Add the integer x to the bitmap
  452. func (rb *Bitmap) Add(x uint32) {
  453. hb := highbits(x)
  454. ra := &rb.highlowcontainer
  455. i := ra.getIndex(hb)
  456. if i >= 0 {
  457. var c container
  458. c = ra.getWritableContainerAtIndex(i).iaddReturnMinimized(lowbits(x))
  459. rb.highlowcontainer.setContainerAtIndex(i, c)
  460. } else {
  461. newac := newArrayContainer()
  462. rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, newac.iaddReturnMinimized(lowbits(x)))
  463. }
  464. }
  465. // add the integer x to the bitmap, return the container and its index
  466. func (rb *Bitmap) addwithptr(x uint32) (int, container) {
  467. hb := highbits(x)
  468. ra := &rb.highlowcontainer
  469. i := ra.getIndex(hb)
  470. var c container
  471. if i >= 0 {
  472. c = ra.getWritableContainerAtIndex(i).iaddReturnMinimized(lowbits(x))
  473. rb.highlowcontainer.setContainerAtIndex(i, c)
  474. return i, c
  475. }
  476. newac := newArrayContainer()
  477. c = newac.iaddReturnMinimized(lowbits(x))
  478. rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, c)
  479. return -i - 1, c
  480. }
  481. // CheckedAdd adds the integer x to the bitmap and return true if it was added (false if the integer was already present)
  482. func (rb *Bitmap) CheckedAdd(x uint32) bool {
  483. // TODO: add unit tests for this method
  484. hb := highbits(x)
  485. i := rb.highlowcontainer.getIndex(hb)
  486. if i >= 0 {
  487. C := rb.highlowcontainer.getWritableContainerAtIndex(i)
  488. oldcard := C.getCardinality()
  489. C = C.iaddReturnMinimized(lowbits(x))
  490. rb.highlowcontainer.setContainerAtIndex(i, C)
  491. return C.getCardinality() > oldcard
  492. }
  493. newac := newArrayContainer()
  494. rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, newac.iaddReturnMinimized(lowbits(x)))
  495. return true
  496. }
  497. // AddInt adds the integer x to the bitmap (convenience method: the parameter is casted to uint32 and we call Add)
  498. func (rb *Bitmap) AddInt(x int) {
  499. rb.Add(uint32(x))
  500. }
  501. // Remove the integer x from the bitmap
  502. func (rb *Bitmap) Remove(x uint32) {
  503. hb := highbits(x)
  504. i := rb.highlowcontainer.getIndex(hb)
  505. if i >= 0 {
  506. c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveReturnMinimized(lowbits(x))
  507. rb.highlowcontainer.setContainerAtIndex(i, c)
  508. if rb.highlowcontainer.getContainerAtIndex(i).getCardinality() == 0 {
  509. rb.highlowcontainer.removeAtIndex(i)
  510. }
  511. }
  512. }
  513. // CheckedRemove removes the integer x from the bitmap and return true if the integer was effectively remove (and false if the integer was not present)
  514. func (rb *Bitmap) CheckedRemove(x uint32) bool {
  515. // TODO: add unit tests for this method
  516. hb := highbits(x)
  517. i := rb.highlowcontainer.getIndex(hb)
  518. if i >= 0 {
  519. C := rb.highlowcontainer.getWritableContainerAtIndex(i)
  520. oldcard := C.getCardinality()
  521. C = C.iremoveReturnMinimized(lowbits(x))
  522. rb.highlowcontainer.setContainerAtIndex(i, C)
  523. if rb.highlowcontainer.getContainerAtIndex(i).getCardinality() == 0 {
  524. rb.highlowcontainer.removeAtIndex(i)
  525. return true
  526. }
  527. return C.getCardinality() < oldcard
  528. }
  529. return false
  530. }
  531. // IsEmpty returns true if the Bitmap is empty (it is faster than doing (GetCardinality() == 0))
  532. func (rb *Bitmap) IsEmpty() bool {
  533. return rb.highlowcontainer.size() == 0
  534. }
  535. // GetCardinality returns the number of integers contained in the bitmap
  536. func (rb *Bitmap) GetCardinality() uint64 {
  537. size := uint64(0)
  538. for _, c := range rb.highlowcontainer.containers {
  539. size += uint64(c.getCardinality())
  540. }
  541. return size
  542. }
  543. // Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be GetCardinality())
  544. func (rb *Bitmap) Rank(x uint32) uint64 {
  545. size := uint64(0)
  546. for i := 0; i < rb.highlowcontainer.size(); i++ {
  547. key := rb.highlowcontainer.getKeyAtIndex(i)
  548. if key > highbits(x) {
  549. return size
  550. }
  551. if key < highbits(x) {
  552. size += uint64(rb.highlowcontainer.getContainerAtIndex(i).getCardinality())
  553. } else {
  554. return size + uint64(rb.highlowcontainer.getContainerAtIndex(i).rank(lowbits(x)))
  555. }
  556. }
  557. return size
  558. }
  559. // Select returns the xth integer in the bitmap
  560. func (rb *Bitmap) Select(x uint32) (uint32, error) {
  561. if rb.GetCardinality() <= uint64(x) {
  562. return 0, fmt.Errorf("can't find %dth integer in a bitmap with only %d items", x, rb.GetCardinality())
  563. }
  564. remaining := x
  565. for i := 0; i < rb.highlowcontainer.size(); i++ {
  566. c := rb.highlowcontainer.getContainerAtIndex(i)
  567. if remaining >= uint32(c.getCardinality()) {
  568. remaining -= uint32(c.getCardinality())
  569. } else {
  570. key := rb.highlowcontainer.getKeyAtIndex(i)
  571. return uint32(key)<<16 + uint32(c.selectInt(uint16(remaining))), nil
  572. }
  573. }
  574. return 0, fmt.Errorf("can't find %dth integer in a bitmap with only %d items", x, rb.GetCardinality())
  575. }
  576. // And computes the intersection between two bitmaps and stores the result in the current bitmap
  577. func (rb *Bitmap) And(x2 *Bitmap) {
  578. pos1 := 0
  579. pos2 := 0
  580. intersectionsize := 0
  581. length1 := rb.highlowcontainer.size()
  582. length2 := x2.highlowcontainer.size()
  583. main:
  584. for {
  585. if pos1 < length1 && pos2 < length2 {
  586. s1 := rb.highlowcontainer.getKeyAtIndex(pos1)
  587. s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
  588. for {
  589. if s1 == s2 {
  590. c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1)
  591. c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
  592. diff := c1.iand(c2)
  593. if diff.getCardinality() > 0 {
  594. rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false)
  595. intersectionsize++
  596. }
  597. pos1++
  598. pos2++
  599. if (pos1 == length1) || (pos2 == length2) {
  600. break main
  601. }
  602. s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
  603. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  604. } else if s1 < s2 {
  605. pos1 = rb.highlowcontainer.advanceUntil(s2, pos1)
  606. if pos1 == length1 {
  607. break main
  608. }
  609. s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
  610. } else { //s1 > s2
  611. pos2 = x2.highlowcontainer.advanceUntil(s1, pos2)
  612. if pos2 == length2 {
  613. break main
  614. }
  615. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  616. }
  617. }
  618. } else {
  619. break
  620. }
  621. }
  622. rb.highlowcontainer.resize(intersectionsize)
  623. }
  624. // OrCardinality returns the cardinality of the union between two bitmaps, bitmaps are not modified
  625. func (rb *Bitmap) OrCardinality(x2 *Bitmap) uint64 {
  626. pos1 := 0
  627. pos2 := 0
  628. length1 := rb.highlowcontainer.size()
  629. length2 := x2.highlowcontainer.size()
  630. answer := uint64(0)
  631. main:
  632. for {
  633. if (pos1 < length1) && (pos2 < length2) {
  634. s1 := rb.highlowcontainer.getKeyAtIndex(pos1)
  635. s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
  636. for {
  637. if s1 < s2 {
  638. answer += uint64(rb.highlowcontainer.getContainerAtIndex(pos1).getCardinality())
  639. pos1++
  640. if pos1 == length1 {
  641. break main
  642. }
  643. s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
  644. } else if s1 > s2 {
  645. answer += uint64(x2.highlowcontainer.getContainerAtIndex(pos2).getCardinality())
  646. pos2++
  647. if pos2 == length2 {
  648. break main
  649. }
  650. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  651. } else {
  652. // TODO: could be faster if we did not have to materialize the container
  653. answer += uint64(rb.highlowcontainer.getContainerAtIndex(pos1).or(x2.highlowcontainer.getContainerAtIndex(pos2)).getCardinality())
  654. pos1++
  655. pos2++
  656. if (pos1 == length1) || (pos2 == length2) {
  657. break main
  658. }
  659. s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
  660. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  661. }
  662. }
  663. } else {
  664. break
  665. }
  666. }
  667. for ; pos1 < length1; pos1++ {
  668. answer += uint64(rb.highlowcontainer.getContainerAtIndex(pos1).getCardinality())
  669. }
  670. for ; pos2 < length2; pos2++ {
  671. answer += uint64(x2.highlowcontainer.getContainerAtIndex(pos2).getCardinality())
  672. }
  673. return answer
  674. }
  675. // AndCardinality returns the cardinality of the intersection between two bitmaps, bitmaps are not modified
  676. func (rb *Bitmap) AndCardinality(x2 *Bitmap) uint64 {
  677. pos1 := 0
  678. pos2 := 0
  679. answer := uint64(0)
  680. length1 := rb.highlowcontainer.size()
  681. length2 := x2.highlowcontainer.size()
  682. main:
  683. for {
  684. if pos1 < length1 && pos2 < length2 {
  685. s1 := rb.highlowcontainer.getKeyAtIndex(pos1)
  686. s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
  687. for {
  688. if s1 == s2 {
  689. c1 := rb.highlowcontainer.getContainerAtIndex(pos1)
  690. c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
  691. answer += uint64(c1.andCardinality(c2))
  692. pos1++
  693. pos2++
  694. if (pos1 == length1) || (pos2 == length2) {
  695. break main
  696. }
  697. s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
  698. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  699. } else if s1 < s2 {
  700. pos1 = rb.highlowcontainer.advanceUntil(s2, pos1)
  701. if pos1 == length1 {
  702. break main
  703. }
  704. s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
  705. } else { //s1 > s2
  706. pos2 = x2.highlowcontainer.advanceUntil(s1, pos2)
  707. if pos2 == length2 {
  708. break main
  709. }
  710. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  711. }
  712. }
  713. } else {
  714. break
  715. }
  716. }
  717. return answer
  718. }
  719. // Intersects checks whether two bitmap intersects, bitmaps are not modified
  720. func (rb *Bitmap) Intersects(x2 *Bitmap) bool {
  721. pos1 := 0
  722. pos2 := 0
  723. length1 := rb.highlowcontainer.size()
  724. length2 := x2.highlowcontainer.size()
  725. main:
  726. for {
  727. if pos1 < length1 && pos2 < length2 {
  728. s1 := rb.highlowcontainer.getKeyAtIndex(pos1)
  729. s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
  730. for {
  731. if s1 == s2 {
  732. c1 := rb.highlowcontainer.getContainerAtIndex(pos1)
  733. c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
  734. if c1.intersects(c2) {
  735. return true
  736. }
  737. pos1++
  738. pos2++
  739. if (pos1 == length1) || (pos2 == length2) {
  740. break main
  741. }
  742. s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
  743. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  744. } else if s1 < s2 {
  745. pos1 = rb.highlowcontainer.advanceUntil(s2, pos1)
  746. if pos1 == length1 {
  747. break main
  748. }
  749. s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
  750. } else { //s1 > s2
  751. pos2 = x2.highlowcontainer.advanceUntil(s1, pos2)
  752. if pos2 == length2 {
  753. break main
  754. }
  755. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  756. }
  757. }
  758. } else {
  759. break
  760. }
  761. }
  762. return false
  763. }
  764. // Xor computes the symmetric difference between two bitmaps and stores the result in the current bitmap
  765. func (rb *Bitmap) Xor(x2 *Bitmap) {
  766. pos1 := 0
  767. pos2 := 0
  768. length1 := rb.highlowcontainer.size()
  769. length2 := x2.highlowcontainer.size()
  770. for {
  771. if (pos1 < length1) && (pos2 < length2) {
  772. s1 := rb.highlowcontainer.getKeyAtIndex(pos1)
  773. s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
  774. if s1 < s2 {
  775. pos1 = rb.highlowcontainer.advanceUntil(s2, pos1)
  776. if pos1 == length1 {
  777. break
  778. }
  779. } else if s1 > s2 {
  780. c := x2.highlowcontainer.getWritableContainerAtIndex(pos2)
  781. rb.highlowcontainer.insertNewKeyValueAt(pos1, x2.highlowcontainer.getKeyAtIndex(pos2), c)
  782. length1++
  783. pos1++
  784. pos2++
  785. } else {
  786. // TODO: couple be computed in-place for reduced memory usage
  787. c := rb.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2))
  788. if c.getCardinality() > 0 {
  789. rb.highlowcontainer.setContainerAtIndex(pos1, c)
  790. pos1++
  791. } else {
  792. rb.highlowcontainer.removeAtIndex(pos1)
  793. length1--
  794. }
  795. pos2++
  796. }
  797. } else {
  798. break
  799. }
  800. }
  801. if pos1 == length1 {
  802. rb.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2)
  803. }
  804. }
  805. // Or computes the union between two bitmaps and stores the result in the current bitmap
  806. func (rb *Bitmap) Or(x2 *Bitmap) {
  807. pos1 := 0
  808. pos2 := 0
  809. length1 := rb.highlowcontainer.size()
  810. length2 := x2.highlowcontainer.size()
  811. main:
  812. for (pos1 < length1) && (pos2 < length2) {
  813. s1 := rb.highlowcontainer.getKeyAtIndex(pos1)
  814. s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
  815. for {
  816. if s1 < s2 {
  817. pos1++
  818. if pos1 == length1 {
  819. break main
  820. }
  821. s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
  822. } else if s1 > s2 {
  823. rb.highlowcontainer.insertNewKeyValueAt(pos1, s2, x2.highlowcontainer.getContainerAtIndex(pos2).clone())
  824. pos1++
  825. length1++
  826. pos2++
  827. if pos2 == length2 {
  828. break main
  829. }
  830. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  831. } else {
  832. rb.highlowcontainer.replaceKeyAndContainerAtIndex(pos1, s1, rb.highlowcontainer.getWritableContainerAtIndex(pos1).ior(x2.highlowcontainer.getContainerAtIndex(pos2)), false)
  833. pos1++
  834. pos2++
  835. if (pos1 == length1) || (pos2 == length2) {
  836. break main
  837. }
  838. s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
  839. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  840. }
  841. }
  842. }
  843. if pos1 == length1 {
  844. rb.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2)
  845. }
  846. }
  847. // AndNot computes the difference between two bitmaps and stores the result in the current bitmap
  848. func (rb *Bitmap) AndNot(x2 *Bitmap) {
  849. pos1 := 0
  850. pos2 := 0
  851. intersectionsize := 0
  852. length1 := rb.highlowcontainer.size()
  853. length2 := x2.highlowcontainer.size()
  854. main:
  855. for {
  856. if pos1 < length1 && pos2 < length2 {
  857. s1 := rb.highlowcontainer.getKeyAtIndex(pos1)
  858. s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
  859. for {
  860. if s1 == s2 {
  861. c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1)
  862. c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
  863. diff := c1.iandNot(c2)
  864. if diff.getCardinality() > 0 {
  865. rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false)
  866. intersectionsize++
  867. }
  868. pos1++
  869. pos2++
  870. if (pos1 == length1) || (pos2 == length2) {
  871. break main
  872. }
  873. s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
  874. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  875. } else if s1 < s2 {
  876. c1 := rb.highlowcontainer.getContainerAtIndex(pos1)
  877. mustCopyOnWrite := rb.highlowcontainer.needsCopyOnWrite(pos1)
  878. rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, c1, mustCopyOnWrite)
  879. intersectionsize++
  880. pos1++
  881. if pos1 == length1 {
  882. break main
  883. }
  884. s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
  885. } else { //s1 > s2
  886. pos2 = x2.highlowcontainer.advanceUntil(s1, pos2)
  887. if pos2 == length2 {
  888. break main
  889. }
  890. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  891. }
  892. }
  893. } else {
  894. break
  895. }
  896. }
  897. // TODO:implement as a copy
  898. for pos1 < length1 {
  899. c1 := rb.highlowcontainer.getContainerAtIndex(pos1)
  900. s1 := rb.highlowcontainer.getKeyAtIndex(pos1)
  901. mustCopyOnWrite := rb.highlowcontainer.needsCopyOnWrite(pos1)
  902. rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, c1, mustCopyOnWrite)
  903. intersectionsize++
  904. pos1++
  905. }
  906. rb.highlowcontainer.resize(intersectionsize)
  907. }
  908. // Or computes the union between two bitmaps and returns the result
  909. func Or(x1, x2 *Bitmap) *Bitmap {
  910. answer := NewBitmap()
  911. pos1 := 0
  912. pos2 := 0
  913. length1 := x1.highlowcontainer.size()
  914. length2 := x2.highlowcontainer.size()
  915. main:
  916. for (pos1 < length1) && (pos2 < length2) {
  917. s1 := x1.highlowcontainer.getKeyAtIndex(pos1)
  918. s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
  919. for {
  920. if s1 < s2 {
  921. answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1)
  922. pos1++
  923. if pos1 == length1 {
  924. break main
  925. }
  926. s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
  927. } else if s1 > s2 {
  928. answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2)
  929. pos2++
  930. if pos2 == length2 {
  931. break main
  932. }
  933. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  934. } else {
  935. answer.highlowcontainer.appendContainer(s1, x1.highlowcontainer.getContainerAtIndex(pos1).or(x2.highlowcontainer.getContainerAtIndex(pos2)), false)
  936. pos1++
  937. pos2++
  938. if (pos1 == length1) || (pos2 == length2) {
  939. break main
  940. }
  941. s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
  942. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  943. }
  944. }
  945. }
  946. if pos1 == length1 {
  947. answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2)
  948. } else if pos2 == length2 {
  949. answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1)
  950. }
  951. return answer
  952. }
  953. // And computes the intersection between two bitmaps and returns the result
  954. func And(x1, x2 *Bitmap) *Bitmap {
  955. answer := NewBitmap()
  956. pos1 := 0
  957. pos2 := 0
  958. length1 := x1.highlowcontainer.size()
  959. length2 := x2.highlowcontainer.size()
  960. main:
  961. for pos1 < length1 && pos2 < length2 {
  962. s1 := x1.highlowcontainer.getKeyAtIndex(pos1)
  963. s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
  964. for {
  965. if s1 == s2 {
  966. C := x1.highlowcontainer.getContainerAtIndex(pos1)
  967. C = C.and(x2.highlowcontainer.getContainerAtIndex(pos2))
  968. if C.getCardinality() > 0 {
  969. answer.highlowcontainer.appendContainer(s1, C, false)
  970. }
  971. pos1++
  972. pos2++
  973. if (pos1 == length1) || (pos2 == length2) {
  974. break main
  975. }
  976. s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
  977. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  978. } else if s1 < s2 {
  979. pos1 = x1.highlowcontainer.advanceUntil(s2, pos1)
  980. if pos1 == length1 {
  981. break main
  982. }
  983. s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
  984. } else { // s1 > s2
  985. pos2 = x2.highlowcontainer.advanceUntil(s1, pos2)
  986. if pos2 == length2 {
  987. break main
  988. }
  989. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  990. }
  991. }
  992. }
  993. return answer
  994. }
  995. // Xor computes the symmetric difference between two bitmaps and returns the result
  996. func Xor(x1, x2 *Bitmap) *Bitmap {
  997. answer := NewBitmap()
  998. pos1 := 0
  999. pos2 := 0
  1000. length1 := x1.highlowcontainer.size()
  1001. length2 := x2.highlowcontainer.size()
  1002. for {
  1003. if (pos1 < length1) && (pos2 < length2) {
  1004. s1 := x1.highlowcontainer.getKeyAtIndex(pos1)
  1005. s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
  1006. if s1 < s2 {
  1007. answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1)
  1008. pos1++
  1009. } else if s1 > s2 {
  1010. answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2)
  1011. pos2++
  1012. } else {
  1013. c := x1.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2))
  1014. if c.getCardinality() > 0 {
  1015. answer.highlowcontainer.appendContainer(s1, c, false)
  1016. }
  1017. pos1++
  1018. pos2++
  1019. }
  1020. } else {
  1021. break
  1022. }
  1023. }
  1024. if pos1 == length1 {
  1025. answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2)
  1026. } else if pos2 == length2 {
  1027. answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1)
  1028. }
  1029. return answer
  1030. }
  1031. // AndNot computes the difference between two bitmaps and returns the result
  1032. func AndNot(x1, x2 *Bitmap) *Bitmap {
  1033. answer := NewBitmap()
  1034. pos1 := 0
  1035. pos2 := 0
  1036. length1 := x1.highlowcontainer.size()
  1037. length2 := x2.highlowcontainer.size()
  1038. main:
  1039. for {
  1040. if pos1 < length1 && pos2 < length2 {
  1041. s1 := x1.highlowcontainer.getKeyAtIndex(pos1)
  1042. s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
  1043. for {
  1044. if s1 < s2 {
  1045. answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1)
  1046. pos1++
  1047. if pos1 == length1 {
  1048. break main
  1049. }
  1050. s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
  1051. } else if s1 == s2 {
  1052. c1 := x1.highlowcontainer.getContainerAtIndex(pos1)
  1053. c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
  1054. diff := c1.andNot(c2)
  1055. if diff.getCardinality() > 0 {
  1056. answer.highlowcontainer.appendContainer(s1, diff, false)
  1057. }
  1058. pos1++
  1059. pos2++
  1060. if (pos1 == length1) || (pos2 == length2) {
  1061. break main
  1062. }
  1063. s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
  1064. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  1065. } else { //s1 > s2
  1066. pos2 = x2.highlowcontainer.advanceUntil(s1, pos2)
  1067. if pos2 == length2 {
  1068. break main
  1069. }
  1070. s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
  1071. }
  1072. }
  1073. } else {
  1074. break
  1075. }
  1076. }
  1077. if pos2 == length2 {
  1078. answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1)
  1079. }
  1080. return answer
  1081. }
  1082. // AddMany add all of the values in dat
  1083. func (rb *Bitmap) AddMany(dat []uint32) {
  1084. if len(dat) == 0 {
  1085. return
  1086. }
  1087. prev := dat[0]
  1088. idx, c := rb.addwithptr(prev)
  1089. for _, i := range dat[1:] {
  1090. if highbits(prev) == highbits(i) {
  1091. c = c.iaddReturnMinimized(lowbits(i))
  1092. rb.highlowcontainer.setContainerAtIndex(idx, c)
  1093. } else {
  1094. idx, c = rb.addwithptr(i)
  1095. }
  1096. prev = i
  1097. }
  1098. }
  1099. // BitmapOf generates a new bitmap filled with the specified integers
  1100. func BitmapOf(dat ...uint32) *Bitmap {
  1101. ans := NewBitmap()
  1102. ans.AddMany(dat)
  1103. return ans
  1104. }
  1105. // Flip negates the bits in the given range (i.e., [rangeStart,rangeEnd)), any integer present in this range and in the bitmap is removed,
  1106. // and any integer present in the range and not in the bitmap is added.
  1107. // The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range
  1108. // while uint64(0x100000000) cannot be represented as a 32-bit value.
  1109. func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) {
  1110. if rangeEnd > MaxUint32+1 {
  1111. panic("rangeEnd > MaxUint32+1")
  1112. }
  1113. if rangeStart > MaxUint32+1 {
  1114. panic("rangeStart > MaxUint32+1")
  1115. }
  1116. if rangeStart >= rangeEnd {
  1117. return
  1118. }
  1119. hbStart := uint32(highbits(uint32(rangeStart)))
  1120. lbStart := uint32(lowbits(uint32(rangeStart)))
  1121. hbLast := uint32(highbits(uint32(rangeEnd - 1)))
  1122. lbLast := uint32(lowbits(uint32(rangeEnd - 1)))
  1123. var max uint32 = maxLowBit
  1124. for hb := hbStart; hb <= hbLast; hb++ {
  1125. var containerStart uint32
  1126. if hb == hbStart {
  1127. containerStart = uint32(lbStart)
  1128. }
  1129. containerLast := max
  1130. if hb == hbLast {
  1131. containerLast = uint32(lbLast)
  1132. }
  1133. i := rb.highlowcontainer.getIndex(uint16(hb))
  1134. if i >= 0 {
  1135. c := rb.highlowcontainer.getWritableContainerAtIndex(i).inot(int(containerStart), int(containerLast)+1)
  1136. if c.getCardinality() > 0 {
  1137. rb.highlowcontainer.setContainerAtIndex(i, c)
  1138. } else {
  1139. rb.highlowcontainer.removeAtIndex(i)
  1140. }
  1141. } else { // *think* the range of ones must never be
  1142. // empty.
  1143. rb.highlowcontainer.insertNewKeyValueAt(-i-1, uint16(hb), rangeOfOnes(int(containerStart), int(containerLast)))
  1144. }
  1145. }
  1146. }
  1147. // FlipInt calls Flip after casting the parameters (convenience method)
  1148. func (rb *Bitmap) FlipInt(rangeStart, rangeEnd int) {
  1149. rb.Flip(uint64(rangeStart), uint64(rangeEnd))
  1150. }
  1151. // AddRange adds the integers in [rangeStart, rangeEnd) to the bitmap.
  1152. // The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range
  1153. // while uint64(0x100000000) cannot be represented as a 32-bit value.
  1154. func (rb *Bitmap) AddRange(rangeStart, rangeEnd uint64) {
  1155. if rangeStart >= rangeEnd {
  1156. return
  1157. }
  1158. if rangeEnd-1 > MaxUint32 {
  1159. panic("rangeEnd-1 > MaxUint32")
  1160. }
  1161. hbStart := uint32(highbits(uint32(rangeStart)))
  1162. lbStart := uint32(lowbits(uint32(rangeStart)))
  1163. hbLast := uint32(highbits(uint32(rangeEnd - 1)))
  1164. lbLast := uint32(lowbits(uint32(rangeEnd - 1)))
  1165. var max uint32 = maxLowBit
  1166. for hb := hbStart; hb <= hbLast; hb++ {
  1167. containerStart := uint32(0)
  1168. if hb == hbStart {
  1169. containerStart = lbStart
  1170. }
  1171. containerLast := max
  1172. if hb == hbLast {
  1173. containerLast = lbLast
  1174. }
  1175. i := rb.highlowcontainer.getIndex(uint16(hb))
  1176. if i >= 0 {
  1177. c := rb.highlowcontainer.getWritableContainerAtIndex(i).iaddRange(int(containerStart), int(containerLast)+1)
  1178. rb.highlowcontainer.setContainerAtIndex(i, c)
  1179. } else { // *think* the range of ones must never be
  1180. // empty.
  1181. rb.highlowcontainer.insertNewKeyValueAt(-i-1, uint16(hb), rangeOfOnes(int(containerStart), int(containerLast)))
  1182. }
  1183. }
  1184. }
  1185. // RemoveRange removes the integers in [rangeStart, rangeEnd) from the bitmap.
  1186. // The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range
  1187. // while uint64(0x100000000) cannot be represented as a 32-bit value.
  1188. func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) {
  1189. if rangeStart >= rangeEnd {
  1190. return
  1191. }
  1192. if rangeEnd-1 > MaxUint32 {
  1193. // logically, we should assume that the user wants to
  1194. // remove all values from rangeStart to infinity
  1195. // see https://github.com/RoaringBitmap/roaring/issues/141
  1196. rangeEnd = uint64(0x100000000)
  1197. }
  1198. hbStart := uint32(highbits(uint32(rangeStart)))
  1199. lbStart := uint32(lowbits(uint32(rangeStart)))
  1200. hbLast := uint32(highbits(uint32(rangeEnd - 1)))
  1201. lbLast := uint32(lowbits(uint32(rangeEnd - 1)))
  1202. var max uint32 = maxLowBit
  1203. if hbStart == hbLast {
  1204. i := rb.highlowcontainer.getIndex(uint16(hbStart))
  1205. if i < 0 {
  1206. return
  1207. }
  1208. c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveRange(int(lbStart), int(lbLast+1))
  1209. if c.getCardinality() > 0 {
  1210. rb.highlowcontainer.setContainerAtIndex(i, c)
  1211. } else {
  1212. rb.highlowcontainer.removeAtIndex(i)
  1213. }
  1214. return
  1215. }
  1216. ifirst := rb.highlowcontainer.getIndex(uint16(hbStart))
  1217. ilast := rb.highlowcontainer.getIndex(uint16(hbLast))
  1218. if ifirst >= 0 {
  1219. if lbStart != 0 {
  1220. c := rb.highlowcontainer.getWritableContainerAtIndex(ifirst).iremoveRange(int(lbStart), int(max+1))
  1221. if c.getCardinality() > 0 {
  1222. rb.highlowcontainer.setContainerAtIndex(ifirst, c)
  1223. ifirst++
  1224. }
  1225. }
  1226. } else {
  1227. ifirst = -ifirst - 1
  1228. }
  1229. if ilast >= 0 {
  1230. if lbLast != max {
  1231. c := rb.highlowcontainer.getWritableContainerAtIndex(ilast).iremoveRange(int(0), int(lbLast+1))
  1232. if c.getCardinality() > 0 {
  1233. rb.highlowcontainer.setContainerAtIndex(ilast, c)
  1234. } else {
  1235. ilast++
  1236. }
  1237. } else {
  1238. ilast++
  1239. }
  1240. } else {
  1241. ilast = -ilast - 1
  1242. }
  1243. rb.highlowcontainer.removeIndexRange(ifirst, ilast)
  1244. }
  1245. // Flip negates the bits in the given range (i.e., [rangeStart,rangeEnd)), any integer present in this range and in the bitmap is removed,
  1246. // and any integer present in the range and not in the bitmap is added, a new bitmap is returned leaving
  1247. // the current bitmap unchanged.
  1248. // The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range
  1249. // while uint64(0x100000000) cannot be represented as a 32-bit value.
  1250. func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap {
  1251. if rangeStart >= rangeEnd {
  1252. return bm.Clone()
  1253. }
  1254. if rangeStart > MaxUint32 {
  1255. panic("rangeStart > MaxUint32")
  1256. }
  1257. if rangeEnd-1 > MaxUint32 {
  1258. panic("rangeEnd-1 > MaxUint32")
  1259. }
  1260. answer := NewBitmap()
  1261. hbStart := uint32(highbits(uint32(rangeStart)))
  1262. lbStart := uint32(lowbits(uint32(rangeStart)))
  1263. hbLast := uint32(highbits(uint32(rangeEnd - 1)))
  1264. lbLast := uint32(lowbits(uint32(rangeEnd - 1)))
  1265. // copy the containers before the active area
  1266. answer.highlowcontainer.appendCopiesUntil(bm.highlowcontainer, uint16(hbStart))
  1267. var max uint32 = maxLowBit
  1268. for hb := hbStart; hb <= hbLast; hb++ {
  1269. var containerStart uint32
  1270. if hb == hbStart {
  1271. containerStart = uint32(lbStart)
  1272. }
  1273. containerLast := max
  1274. if hb == hbLast {
  1275. containerLast = uint32(lbLast)
  1276. }
  1277. i := bm.highlowcontainer.getIndex(uint16(hb))
  1278. j := answer.highlowcontainer.getIndex(uint16(hb))
  1279. if i >= 0 {
  1280. c := bm.highlowcontainer.getContainerAtIndex(i).not(int(containerStart), int(containerLast)+1)
  1281. if c.getCardinality() > 0 {
  1282. answer.highlowcontainer.insertNewKeyValueAt(-j-1, uint16(hb), c)
  1283. }
  1284. } else { // *think* the range of ones must never be
  1285. // empty.
  1286. answer.highlowcontainer.insertNewKeyValueAt(-j-1, uint16(hb),
  1287. rangeOfOnes(int(containerStart), int(containerLast)))
  1288. }
  1289. }
  1290. // copy the containers after the active area.
  1291. answer.highlowcontainer.appendCopiesAfter(bm.highlowcontainer, uint16(hbLast))
  1292. return answer
  1293. }
  1294. // SetCopyOnWrite sets this bitmap to use copy-on-write so that copies are fast and memory conscious
  1295. // if the parameter is true, otherwise we leave the default where hard copies are made
  1296. // (copy-on-write requires extra care in a threaded context).
  1297. // Calling SetCopyOnWrite(true) on a bitmap created with FromBuffer is unsafe.
  1298. func (rb *Bitmap) SetCopyOnWrite(val bool) {
  1299. rb.highlowcontainer.copyOnWrite = val
  1300. }
  1301. // GetCopyOnWrite gets this bitmap's copy-on-write property
  1302. func (rb *Bitmap) GetCopyOnWrite() (val bool) {
  1303. return rb.highlowcontainer.copyOnWrite
  1304. }
  1305. // CloneCopyOnWriteContainers clones all containers which have
  1306. // needCopyOnWrite set to true.
  1307. // This can be used to make sure it is safe to munmap a []byte
  1308. // that the roaring array may still have a reference to, after
  1309. // calling FromBuffer.
  1310. // More generally this function is useful if you call FromBuffer
  1311. // to construct a bitmap with a backing array buf
  1312. // and then later discard the buf array. Note that you should call
  1313. // CloneCopyOnWriteContainers on all bitmaps that were derived
  1314. // from the 'FromBuffer' bitmap since they map have dependencies
  1315. // on the buf array as well.
  1316. func (rb *Bitmap) CloneCopyOnWriteContainers() {
  1317. rb.highlowcontainer.cloneCopyOnWriteContainers()
  1318. }
  1319. // FlipInt calls Flip after casting the parameters (convenience method)
  1320. func FlipInt(bm *Bitmap, rangeStart, rangeEnd int) *Bitmap {
  1321. return Flip(bm, uint64(rangeStart), uint64(rangeEnd))
  1322. }
  1323. // Statistics provides details on the container types in use.
  1324. type Statistics struct {
  1325. Cardinality uint64
  1326. Containers uint64
  1327. ArrayContainers uint64
  1328. ArrayContainerBytes uint64
  1329. ArrayContainerValues uint64
  1330. BitmapContainers uint64
  1331. BitmapContainerBytes uint64
  1332. BitmapContainerValues uint64
  1333. RunContainers uint64
  1334. RunContainerBytes uint64
  1335. RunContainerValues uint64
  1336. }
  1337. // Stats returns details on container type usage in a Statistics struct.
  1338. func (rb *Bitmap) Stats() Statistics {
  1339. stats := Statistics{}
  1340. stats.Containers = uint64(len(rb.highlowcontainer.containers))
  1341. for _, c := range rb.highlowcontainer.containers {
  1342. stats.Cardinality += uint64(c.getCardinality())
  1343. switch c.(type) {
  1344. case *arrayContainer:
  1345. stats.ArrayContainers++
  1346. stats.ArrayContainerBytes += uint64(c.getSizeInBytes())
  1347. stats.ArrayContainerValues += uint64(c.getCardinality())
  1348. case *bitmapContainer:
  1349. stats.BitmapContainers++
  1350. stats.BitmapContainerBytes += uint64(c.getSizeInBytes())
  1351. stats.BitmapContainerValues += uint64(c.getCardinality())
  1352. case *runContainer16:
  1353. stats.RunContainers++
  1354. stats.RunContainerBytes += uint64(c.getSizeInBytes())
  1355. stats.RunContainerValues += uint64(c.getCardinality())
  1356. }
  1357. }
  1358. return stats
  1359. }