diff options
author | Mura Li <typeless@users.noreply.github.com> | 2019-11-27 17:23:33 +0800 |
---|---|---|
committer | Lauris BH <lauris@nix.lv> | 2019-11-27 11:23:33 +0200 |
commit | 9591185c8f45fefb39e84aa465a943b24ad60a26 (patch) | |
tree | c46c697093c98eb5404aa7e2519fbe4ba1a2ad47 /vendor/github.com/RoaringBitmap | |
parent | b50dee5a61b3959bb3606973d359741f0ca9af9a (diff) | |
download | gitea-9591185c8f45fefb39e84aa465a943b24ad60a26.tar.gz gitea-9591185c8f45fefb39e84aa465a943b24ad60a26.zip |
Upgrade blevesearch to v0.8.1 (#9177)
For #1441
https://github.com/blevesearch/bleve/commit/a91b427b59b893f112021841ba7370d285f8426f
Diffstat (limited to 'vendor/github.com/RoaringBitmap')
32 files changed, 1815 insertions, 4197 deletions
diff --git a/vendor/github.com/RoaringBitmap/roaring/.drone.yml b/vendor/github.com/RoaringBitmap/roaring/.drone.yml new file mode 100644 index 0000000000..698cd0e7a7 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/.drone.yml @@ -0,0 +1,20 @@ +kind: pipeline +name: default + +workspace: + base: /go + path: src/github.com/RoaringBitmap/roaring + +steps: +- name: test + image: golang + commands: + - go get -t + - go test + - go test -race -run TestConcurrent* + - go build -tags appengine + - go test -tags appengine + - GOARCH=386 go build + - GOARCH=386 go test + - GOARCH=arm go build + - GOARCH=arm64 go build diff --git a/vendor/github.com/RoaringBitmap/roaring/.travis.yml b/vendor/github.com/RoaringBitmap/roaring/.travis.yml index 1fdcf3e057..8839c14fd0 100644 --- a/vendor/github.com/RoaringBitmap/roaring/.travis.yml +++ b/vendor/github.com/RoaringBitmap/roaring/.travis.yml @@ -8,10 +8,12 @@ install: notifications: email: false go: -- 1.7.x -- 1.8.x -- 1.9.x -- 1.10.x +- "1.7.x" +- "1.8.x" +- "1.9.x" +- "1.10.x" +- "1.11.x" +- "1.12.x" - tip # whitelist @@ -21,10 +23,14 @@ branches: script: - goveralls -v -service travis-ci -ignore arraycontainer_gen.go,bitmapcontainer_gen.go,rle16_gen.go,rle_gen.go,roaringarray_gen.go,rle.go || go test - go test -race -run TestConcurrent* +- go build -tags appengine +- go test -tags appengine - GOARCH=arm64 go build - GOARCH=386 go build - GOARCH=386 go test - GOARCH=arm go build +- GOARCH=arm64 go build + matrix: allow_failures: - go: tip diff --git a/vendor/github.com/RoaringBitmap/roaring/AUTHORS b/vendor/github.com/RoaringBitmap/roaring/AUTHORS index 08c074047f..26ec99de9d 100644 --- a/vendor/github.com/RoaringBitmap/roaring/AUTHORS +++ b/vendor/github.com/RoaringBitmap/roaring/AUTHORS @@ -7,4 +7,5 @@ Bob Potter (@bpot), Tyson Maly (@tvmaly), Will Glynn (@willglynn), Brent Pedersen (@brentp) -Maciej Biłas (@maciej) +Maciej Biłas (@maciej), +Joe Nall (@joenall) diff --git a/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS b/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS index 70b4735dad..b1e3a379f0 100644 --- a/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS +++ b/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS @@ -9,4 +9,8 @@ Will Glynn (@willglynn), Brent Pedersen (@brentp), Jason E. Aten (@glycerine), Vali Malinoiu (@0x4139), -Forud Ghafouri (@fzerorubigd)
\ No newline at end of file +Forud Ghafouri (@fzerorubigd), +Joe Nall (@joenall), +(@fredim), +Edd Robinson (@e-dard), +Alexander Petrov (@alldroll) diff --git a/vendor/github.com/RoaringBitmap/roaring/LICENSE b/vendor/github.com/RoaringBitmap/roaring/LICENSE index aff5f9999b..3ccdd00084 100644 --- a/vendor/github.com/RoaringBitmap/roaring/LICENSE +++ b/vendor/github.com/RoaringBitmap/roaring/LICENSE @@ -200,3 +200,36 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +================================================================================ + +Portions of runcontainer.go are from the Go standard library, which is licensed +under: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/RoaringBitmap/roaring/Makefile b/vendor/github.com/RoaringBitmap/roaring/Makefile index d5259b4c9e..906bd72569 100644 --- a/vendor/github.com/RoaringBitmap/roaring/Makefile +++ b/vendor/github.com/RoaringBitmap/roaring/Makefile @@ -1,4 +1,4 @@ -.PHONY: help all test format fmtcheck vet lint qa deps clean nuke rle backrle ser fetch-real-roaring-datasets +.PHONY: help all test format fmtcheck vet lint qa deps clean nuke ser fetch-real-roaring-datasets @@ -63,7 +63,7 @@ qa: fmtcheck test vet lint # Get the dependencies deps: - GOPATH=$(GOPATH) go get github.com/smartystreets/goconvey/convey + GOPATH=$(GOPATH) go get github.com/stretchr/testify GOPATH=$(GOPATH) go get github.com/willf/bitset GOPATH=$(GOPATH) go get github.com/golang/lint/golint GOPATH=$(GOPATH) go get github.com/mschoch/smat @@ -97,18 +97,8 @@ nuke: rm -rf ./target GOPATH=$(GOPATH) go clean -i ./... -rle: - cp rle.go rle16.go - perl -pi -e 's/32/16/g' rle16.go - cp rle_test.go rle16_test.go - perl -pi -e 's/32/16/g' rle16_test.go -backrle: - cp rle16.go rle.go - perl -pi -e 's/16/32/g' rle.go - perl -pi -e 's/2032/2016/g' rle.go - -ser: rle +ser: go generate cover: diff --git a/vendor/github.com/RoaringBitmap/roaring/README.md b/vendor/github.com/RoaringBitmap/roaring/README.md index 2c096ce8e6..b711d09ec2 100644 --- a/vendor/github.com/RoaringBitmap/roaring/README.md +++ b/vendor/github.com/RoaringBitmap/roaring/README.md @@ -1,4 +1,5 @@ roaring [![Build Status](https://travis-ci.org/RoaringBitmap/roaring.png)](https://travis-ci.org/RoaringBitmap/roaring) [![Coverage Status](https://coveralls.io/repos/github/RoaringBitmap/roaring/badge.svg?branch=master)](https://coveralls.io/github/RoaringBitmap/roaring?branch=master) [![GoDoc](https://godoc.org/github.com/RoaringBitmap/roaring?status.svg)](https://godoc.org/github.com/RoaringBitmap/roaring) [![Go Report Card](https://goreportcard.com/badge/RoaringBitmap/roaring)](https://goreportcard.com/report/github.com/RoaringBitmap/roaring) +[![Build Status](https://cloud.drone.io/api/badges/RoaringBitmap/roaring/status.svg)](https://cloud.drone.io/RoaringBitmap/roaring) ============= This is a go version of the Roaring bitmap data structure. @@ -6,12 +7,12 @@ This is a go version of the Roaring bitmap data structure. Roaring bitmaps are used by several major systems such as [Apache Lucene][lucene] and derivative systems such as [Solr][solr] and -[Elasticsearch][elasticsearch], [Metamarkets' Druid][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [Cloud Torrent][cloudtorrent], [Whoosh][whoosh], [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin]. +[Elasticsearch][elasticsearch], [Apache Druid (Incubating)][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [Cloud Torrent][cloudtorrent], [Whoosh][whoosh], [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin]. [lucene]: https://lucene.apache.org/ [solr]: https://lucene.apache.org/solr/ [elasticsearch]: https://www.elastic.co/products/elasticsearch -[druid]: http://druid.io/ +[druid]: https://druid.apache.org/ [spark]: https://spark.apache.org/ [opensearchserver]: http://www.opensearchserver.com [cloudtorrent]: https://github.com/jpillora/cloud-torrent @@ -61,7 +62,6 @@ http://arxiv.org/abs/1402.6407 This paper used data from http://lemire.me/data/r Dependencies are fetched automatically by giving the `-t` flag to `go get`. they include - - github.com/smartystreets/goconvey/convey - github.com/willf/bitset - github.com/mschoch/smat - github.com/glycerine/go-unsnap-stream @@ -133,6 +133,7 @@ func main() { if rb1.Equals(newrb) { fmt.Println("I wrote the content to a byte stream and read it back.") } + // you can iterate over bitmaps using ReverseIterator(), Iterator, ManyIterator() } ``` @@ -206,7 +207,7 @@ You can use roaring with gore: - go get -u github.com/motemen/gore - Make sure that ``$GOPATH/bin`` is in your ``$PATH``. -- go get github/RoaringBitmap/roaring +- go get github.com/RoaringBitmap/roaring ```go $ gore diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go index c395868210..621616f5df 100644 --- a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go +++ b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go @@ -24,12 +24,16 @@ func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uin } } -func (ac *arrayContainer) getShortIterator() shortIterable { +func (ac *arrayContainer) getShortIterator() shortPeekable { return &shortIterator{ac.content, 0} } +func (ac *arrayContainer) getReverseIterator() shortIterable { + return &reverseIterator{ac.content, len(ac.content) - 1} +} + func (ac *arrayContainer) getManyIterator() manyIterable { - return &manyIterator{ac.content, 0} + return &shortIterator{ac.content, 0} } func (ac *arrayContainer) minimum() uint16 { @@ -115,7 +119,6 @@ func (ac *arrayContainer) iremoveRange(firstOfRange, endx int) container { // flip the values in the range [firstOfRange,endx) func (ac *arrayContainer) not(firstOfRange, endx int) container { if firstOfRange >= endx { - //p("arrayContainer.not(): exiting early with ac.clone()") return ac.clone() } return ac.notClose(firstOfRange, endx-1) // remove everything in [firstOfRange,endx-1] @@ -124,18 +127,15 @@ func (ac *arrayContainer) not(firstOfRange, endx int) container { // flip the values in the range [firstOfRange,lastOfRange] func (ac *arrayContainer) notClose(firstOfRange, lastOfRange int) container { if firstOfRange > lastOfRange { // unlike add and remove, not uses an inclusive range [firstOfRange,lastOfRange] - //p("arrayContainer.notClose(): exiting early with ac.clone()") return ac.clone() } // determine the span of array indices to be affected^M startIndex := binarySearch(ac.content, uint16(firstOfRange)) - //p("startIndex=%v", startIndex) if startIndex < 0 { startIndex = -startIndex - 1 } lastIndex := binarySearch(ac.content, uint16(lastOfRange)) - //p("lastIndex=%v", lastIndex) if lastIndex < 0 { lastIndex = -lastIndex - 2 } @@ -144,9 +144,7 @@ func (ac *arrayContainer) notClose(firstOfRange, lastOfRange int) container { newValuesInRange := spanToBeFlipped - currentValuesInRange cardinalityChange := newValuesInRange - currentValuesInRange newCardinality := len(ac.content) + cardinalityChange - //p("new card is %v", newCardinality) if newCardinality > arrayDefaultMaxSize { - //p("new card over arrayDefaultMaxSize, so returning bitmap") return ac.toBitmapContainer().not(firstOfRange, lastOfRange+1) } answer := newArrayContainer() @@ -503,7 +501,6 @@ func (ac *arrayContainer) lazyorArray(value2 *arrayContainer) container { } func (ac *arrayContainer) and(a container) container { - //p("ac.and() called") switch x := a.(type) { case *arrayContainer: return ac.andArray(x) @@ -550,7 +547,7 @@ func (ac *arrayContainer) iand(a container) container { return ac.iandBitmap(x) case *runContainer16: if x.isFull() { - return ac.clone() + return ac } return x.andArray(ac) } @@ -722,7 +719,6 @@ func (ac *arrayContainer) inot(firstOfRange, endx int) container { // flip the values in the range [firstOfRange,lastOfRange] func (ac *arrayContainer) inotClose(firstOfRange, lastOfRange int) container { - //p("ac.inotClose() starting") if firstOfRange > lastOfRange { // unlike add and remove, not uses an inclusive range [firstOfRange,lastOfRange] return ac } @@ -745,7 +741,6 @@ func (ac *arrayContainer) inotClose(firstOfRange, lastOfRange int) container { if cardinalityChange > 0 { if newCardinality > len(ac.content) { if newCardinality > arrayDefaultMaxSize { - //p("ac.inotClose() converting to bitmap and doing inot there") bcRet := ac.toBitmapContainer() bcRet.inot(firstOfRange, lastOfRange+1) *ac = *bcRet.toArrayContainer() @@ -766,7 +761,6 @@ func (ac *arrayContainer) inotClose(firstOfRange, lastOfRange int) container { } } ac.content = ac.content[:newCardinality] - //p("bottom of ac.inotClose(): returning ac") return ac } @@ -958,3 +952,17 @@ func (ac *arrayContainer) toEfficientContainer() container { func (ac *arrayContainer) containerType() contype { return arrayContype } + +func (ac *arrayContainer) addOffset(x uint16) []container { + low := &arrayContainer{} + high := &arrayContainer{} + for _, val := range ac.content { + y := uint32(val) + uint32(x) + if highbits(y) > 0 { + high.content = append(high.content, lowbits(y)) + } else { + low.content = append(low.content, lowbits(y)) + } + } + return []container{low, high} +} diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go index cba6e53e30..6ee670ee51 100644 --- a/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go +++ b/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go @@ -6,7 +6,7 @@ package roaring import "github.com/tinylib/msgp/msgp" -// DecodeMsg implements msgp.Decodable +// Deprecated: DecodeMsg implements msgp.Decodable func (z *arrayContainer) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field @@ -49,7 +49,7 @@ func (z *arrayContainer) DecodeMsg(dc *msgp.Reader) (err error) { return } -// EncodeMsg implements msgp.Encodable +// Deprecated: EncodeMsg implements msgp.Encodable func (z *arrayContainer) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 1 // write "content" @@ -70,7 +70,7 @@ func (z *arrayContainer) EncodeMsg(en *msgp.Writer) (err error) { return } -// MarshalMsg implements msgp.Marshaler +// Deprecated: MarshalMsg implements msgp.Marshaler func (z *arrayContainer) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 1 @@ -83,7 +83,7 @@ func (z *arrayContainer) MarshalMsg(b []byte) (o []byte, err error) { return } -// UnmarshalMsg implements msgp.Unmarshaler +// Deprecated: UnmarshalMsg implements msgp.Unmarshaler func (z *arrayContainer) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field @@ -127,7 +127,7 @@ func (z *arrayContainer) UnmarshalMsg(bts []byte) (o []byte, err error) { return } -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *arrayContainer) Msgsize() (s int) { s = 1 + 8 + msgp.ArrayHeaderSize + (len(z.content) * (msgp.Uint16Size)) return diff --git a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go index 5e58b31f2b..e749721bb2 100644 --- a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go +++ b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go @@ -110,14 +110,54 @@ func (bcsi *bitmapContainerShortIterator) hasNext() bool { return bcsi.i >= 0 } +func (bcsi *bitmapContainerShortIterator) peekNext() uint16 { + return uint16(bcsi.i) +} + +func (bcsi *bitmapContainerShortIterator) advanceIfNeeded(minval uint16) { + if bcsi.hasNext() && bcsi.peekNext() < minval { + bcsi.i = bcsi.ptr.NextSetBit(int(minval)) + } +} + func newBitmapContainerShortIterator(a *bitmapContainer) *bitmapContainerShortIterator { return &bitmapContainerShortIterator{a, a.NextSetBit(0)} } -func (bc *bitmapContainer) getShortIterator() shortIterable { +func (bc *bitmapContainer) getShortIterator() shortPeekable { return newBitmapContainerShortIterator(bc) } +type reverseBitmapContainerShortIterator struct { + ptr *bitmapContainer + i int +} + +func (bcsi *reverseBitmapContainerShortIterator) next() uint16 { + if bcsi.i == -1 { + panic("reverseBitmapContainerShortIterator.next() going beyond what is available") + } + + j := bcsi.i + bcsi.i = bcsi.ptr.PrevSetBit(bcsi.i - 1) + return uint16(j) +} + +func (bcsi *reverseBitmapContainerShortIterator) hasNext() bool { + return bcsi.i >= 0 +} + +func newReverseBitmapContainerShortIterator(a *bitmapContainer) *reverseBitmapContainerShortIterator { + if a.cardinality == 0 { + return &reverseBitmapContainerShortIterator{a, -1} + } + return &reverseBitmapContainerShortIterator{a, int(a.maximum())} +} + +func (bc *bitmapContainer) getReverseIterator() shortIterable { + return newReverseBitmapContainerShortIterator(bc) +} + type bitmapContainerManyIterator struct { ptr *bitmapContainer base int @@ -131,7 +171,7 @@ func (bcmi *bitmapContainerManyIterator) nextMany(hs uint32, buf []uint32) int { for n < len(buf) { if bitset == 0 { - base += 1 + base++ if base >= len(bcmi.ptr.bitmap) { bcmi.base = base bcmi.bitset = bitset @@ -177,16 +217,13 @@ func bitmapContainerSizeInBytes() int { func bitmapEquals(a, b []uint64) bool { if len(a) != len(b) { - //p("bitmaps differ on length. len(a)=%v; len(b)=%v", len(a), len(b)) return false } for i, v := range a { if v != b[i] { - //p("bitmaps differ on element i=%v", i) return false } } - //p("bitmapEquals returning true") return true } @@ -209,9 +246,7 @@ func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask ui func (bc *bitmapContainer) equals(o container) bool { srb, ok := o.(*bitmapContainer) if ok { - //p("bitmapContainers.equals: both are bitmapContainers") if srb.cardinality != bc.cardinality { - //p("bitmapContainers.equals: card differs: %v vs %v", srb.cardinality, bc.cardinality) return false } return bitmapEquals(bc.bitmap, srb.bitmap) @@ -261,12 +296,6 @@ func (bc *bitmapContainer) iremoveReturnMinimized(i uint16) container { // iremove returns true if i was found. func (bc *bitmapContainer) iremove(i uint16) bool { - /* branchless code - w := bc.bitmap[i>>6] - mask := uint64(1) << (i % 64) - neww := w &^ mask - bc.cardinality -= int((w ^ neww) >> (i % 64)) - bc.bitmap[i>>6] = neww */ if bc.contains(i) { bc.cardinality-- bc.bitmap[i/64] &^= (uint64(1) << (i % 64)) @@ -306,14 +335,10 @@ func (bc *bitmapContainer) iremoveRange(firstOfRange, lastOfRange int) container // flip all values in range [firstOfRange,endx) func (bc *bitmapContainer) inot(firstOfRange, endx int) container { - p("bc.inot() called with [%v, %v)", firstOfRange, endx) if endx-firstOfRange == maxCapacity { - //p("endx-firstOfRange == maxCapacity") flipBitmapRange(bc.bitmap, firstOfRange, endx) bc.cardinality = maxCapacity - bc.cardinality - //p("bc.cardinality is now %v", bc.cardinality) } else if endx-firstOfRange > maxCapacity/2 { - //p("endx-firstOfRange > maxCapacity/2") flipBitmapRange(bc.bitmap, firstOfRange, endx) bc.computeCardinality() } else { @@ -517,11 +542,31 @@ func (bc *bitmapContainer) iorBitmap(value2 *bitmapContainer) container { func (bc *bitmapContainer) lazyIORArray(value2 *arrayContainer) container { answer := bc c := value2.getCardinality() - for k := 0; k < c; k++ { + for k := 0; k+3 < c; k += 4 { + content := (*[4]uint16)(unsafe.Pointer(&value2.content[k])) + vc0 := content[0] + i0 := uint(vc0) >> 6 + answer.bitmap[i0] = answer.bitmap[i0] | (uint64(1) << (vc0 % 64)) + + vc1 := content[1] + i1 := uint(vc1) >> 6 + answer.bitmap[i1] = answer.bitmap[i1] | (uint64(1) << (vc1 % 64)) + + vc2 := content[2] + i2 := uint(vc2) >> 6 + answer.bitmap[i2] = answer.bitmap[i2] | (uint64(1) << (vc2 % 64)) + + vc3 := content[3] + i3 := uint(vc3) >> 6 + answer.bitmap[i3] = answer.bitmap[i3] | (uint64(1) << (vc3 % 64)) + } + + for k := c &^ 3; k < c; k++ { vc := value2.content[k] i := uint(vc) >> 6 answer.bitmap[i] = answer.bitmap[i] | (uint64(1) << (vc % 64)) } + answer.cardinality = invalidCardinality return answer } @@ -789,8 +834,6 @@ func (bc *bitmapContainer) andNotRun16(rc *runContainer16) container { } func (bc *bitmapContainer) iandNot(a container) container { - //p("bitmapContainer.iandNot() starting") - switch x := a.(type) { case *arrayContainer: return bc.iandNotArray(x) @@ -844,12 +887,15 @@ func (bc *bitmapContainer) andNotBitmap(value2 *bitmapContainer) container { return ac } -func (bc *bitmapContainer) iandNotBitmapSurely(value2 *bitmapContainer) *bitmapContainer { +func (bc *bitmapContainer) iandNotBitmapSurely(value2 *bitmapContainer) container { newCardinality := int(popcntMaskSlice(bc.bitmap, value2.bitmap)) for k := 0; k < len(bc.bitmap); k++ { bc.bitmap[k] = bc.bitmap[k] &^ value2.bitmap[k] } bc.cardinality = newCardinality + if bc.getCardinality() <= arrayDefaultMaxSize { + return bc.toArrayContainer() + } return bc } @@ -917,6 +963,32 @@ func (bc *bitmapContainer) NextSetBit(i int) int { return -1 } +func (bc *bitmapContainer) PrevSetBit(i int) int { + if i < 0 { + return -1 + } + x := i / 64 + if x >= len(bc.bitmap) { + return -1 + } + + w := bc.bitmap[x] + + b := i % 64 + + w = w << uint(63-b) + if w != 0 { + return i - countLeadingZeros(w) + } + x-- + for ; x >= 0; x-- { + if bc.bitmap[x] != 0 { + return (x * 64) + 63 - countLeadingZeros(bc.bitmap[x]) + } + } + return -1 +} + // reference the java implementation // https://github.com/RoaringBitmap/RoaringBitmap/blob/master/src/main/java/org/roaringbitmap/BitmapContainer.java#L875-L892 // @@ -980,3 +1052,35 @@ func newBitmapContainerFromRun(rc *runContainer16) *bitmapContainer { func (bc *bitmapContainer) containerType() contype { return bitmapContype } + +func (bc *bitmapContainer) addOffset(x uint16) []container { + low := newBitmapContainer() + high := newBitmapContainer() + b := uint32(x) >> 6 + i := uint32(x) % 64 + end := uint32(1024) - b + if i == 0 { + copy(low.bitmap[b:], bc.bitmap[:end]) + copy(high.bitmap[:b], bc.bitmap[end:]) + } else { + low.bitmap[b] = bc.bitmap[0] << i + for k := uint32(1); k < end; k++ { + newval := bc.bitmap[k] << i + if newval == 0 { + newval = bc.bitmap[k-1] >> (64 - i) + } + low.bitmap[b+k] = newval + } + for k := end; k < 1024; k++ { + newval := bc.bitmap[k] << i + if newval == 0 { + newval = bc.bitmap[k-1] >> (64 - i) + } + high.bitmap[k-end] = newval + } + high.bitmap[b] = bc.bitmap[1023] >> (64 - i) + } + low.computeCardinality() + high.computeCardinality() + return []container{low, high} +} diff --git a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go index f6c053e650..9b5a465f38 100644 --- a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go +++ b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go @@ -6,7 +6,7 @@ package roaring import "github.com/tinylib/msgp/msgp" -// DecodeMsg implements msgp.Decodable +// Deprecated: DecodeMsg implements msgp.Decodable func (z *bitmapContainer) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field @@ -54,7 +54,7 @@ func (z *bitmapContainer) DecodeMsg(dc *msgp.Reader) (err error) { return } -// EncodeMsg implements msgp.Encodable +// Deprecated: EncodeMsg implements msgp.Encodable func (z *bitmapContainer) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 2 // write "cardinality" @@ -84,7 +84,7 @@ func (z *bitmapContainer) EncodeMsg(en *msgp.Writer) (err error) { return } -// MarshalMsg implements msgp.Marshaler +// Deprecated: MarshalMsg implements msgp.Marshaler func (z *bitmapContainer) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 2 @@ -100,7 +100,7 @@ func (z *bitmapContainer) MarshalMsg(b []byte) (o []byte, err error) { return } -// UnmarshalMsg implements msgp.Unmarshaler +// Deprecated: UnmarshalMsg implements msgp.Unmarshaler func (z *bitmapContainer) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field @@ -149,13 +149,13 @@ func (z *bitmapContainer) UnmarshalMsg(bts []byte) (o []byte, err error) { return } -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *bitmapContainer) Msgsize() (s int) { s = 1 + 12 + msgp.IntSize + 7 + msgp.ArrayHeaderSize + (len(z.bitmap) * (msgp.Uint64Size)) return } -// DecodeMsg implements msgp.Decodable +// Deprecated: DecodeMsg implements msgp.Decodable func (z *bitmapContainerShortIterator) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field @@ -239,7 +239,7 @@ func (z *bitmapContainerShortIterator) DecodeMsg(dc *msgp.Reader) (err error) { return } -// EncodeMsg implements msgp.Encodable +// Deprecated: EncodeMsg implements msgp.Encodable func (z *bitmapContainerShortIterator) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 2 // write "ptr" @@ -291,7 +291,7 @@ func (z *bitmapContainerShortIterator) EncodeMsg(en *msgp.Writer) (err error) { return } -// MarshalMsg implements msgp.Marshaler +// Deprecated: MarshalMsg implements msgp.Marshaler func (z *bitmapContainerShortIterator) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 2 @@ -317,7 +317,7 @@ func (z *bitmapContainerShortIterator) MarshalMsg(b []byte) (o []byte, err error return } -// UnmarshalMsg implements msgp.Unmarshaler +// Deprecated: UnmarshalMsg implements msgp.Unmarshaler func (z *bitmapContainerShortIterator) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field @@ -402,7 +402,7 @@ func (z *bitmapContainerShortIterator) UnmarshalMsg(bts []byte) (o []byte, err e return } -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *bitmapContainerShortIterator) Msgsize() (s int) { s = 1 + 4 if z.ptr == nil { diff --git a/vendor/github.com/RoaringBitmap/roaring/byte_input.go b/vendor/github.com/RoaringBitmap/roaring/byte_input.go new file mode 100644 index 0000000000..f7a98a1d40 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/byte_input.go @@ -0,0 +1,161 @@ +package roaring + +import ( + "encoding/binary" + "io" +) + +type byteInput interface { + // next returns a slice containing the next n bytes from the buffer, + // advancing the buffer as if the bytes had been returned by Read. + next(n int) ([]byte, error) + // readUInt32 reads uint32 with LittleEndian order + readUInt32() (uint32, error) + // readUInt16 reads uint16 with LittleEndian order + readUInt16() (uint16, error) + // getReadBytes returns read bytes + getReadBytes() int64 + // skipBytes skips exactly n bytes + skipBytes(n int) error +} + +func newByteInputFromReader(reader io.Reader) byteInput { + return &byteInputAdapter{ + r: reader, + readBytes: 0, + } +} + +func newByteInput(buf []byte) byteInput { + return &byteBuffer{ + buf: buf, + off: 0, + } +} + +type byteBuffer struct { + buf []byte + off int +} + +// next returns a slice containing the next n bytes from the reader +// If there are fewer bytes than the given n, io.ErrUnexpectedEOF will be returned +func (b *byteBuffer) next(n int) ([]byte, error) { + m := len(b.buf) - b.off + + if n > m { + return nil, io.ErrUnexpectedEOF + } + + data := b.buf[b.off : b.off+n] + b.off += n + + return data, nil +} + +// readUInt32 reads uint32 with LittleEndian order +func (b *byteBuffer) readUInt32() (uint32, error) { + if len(b.buf)-b.off < 4 { + return 0, io.ErrUnexpectedEOF + } + + v := binary.LittleEndian.Uint32(b.buf[b.off:]) + b.off += 4 + + return v, nil +} + +// readUInt16 reads uint16 with LittleEndian order +func (b *byteBuffer) readUInt16() (uint16, error) { + if len(b.buf)-b.off < 2 { + return 0, io.ErrUnexpectedEOF + } + + v := binary.LittleEndian.Uint16(b.buf[b.off:]) + b.off += 2 + + return v, nil +} + +// getReadBytes returns read bytes +func (b *byteBuffer) getReadBytes() int64 { + return int64(b.off) +} + +// skipBytes skips exactly n bytes +func (b *byteBuffer) skipBytes(n int) error { + m := len(b.buf) - b.off + + if n > m { + return io.ErrUnexpectedEOF + } + + b.off += n + + return nil +} + +// reset resets the given buffer with a new byte slice +func (b *byteBuffer) reset(buf []byte) { + b.buf = buf + b.off = 0 +} + +type byteInputAdapter struct { + r io.Reader + readBytes int +} + +// next returns a slice containing the next n bytes from the buffer, +// advancing the buffer as if the bytes had been returned by Read. +func (b *byteInputAdapter) next(n int) ([]byte, error) { + buf := make([]byte, n) + m, err := io.ReadAtLeast(b.r, buf, n) + b.readBytes += m + + if err != nil { + return nil, err + } + + return buf, nil +} + +// readUInt32 reads uint32 with LittleEndian order +func (b *byteInputAdapter) readUInt32() (uint32, error) { + buf, err := b.next(4) + + if err != nil { + return 0, err + } + + return binary.LittleEndian.Uint32(buf), nil +} + +// readUInt16 reads uint16 with LittleEndian order +func (b *byteInputAdapter) readUInt16() (uint16, error) { + buf, err := b.next(2) + + if err != nil { + return 0, err + } + + return binary.LittleEndian.Uint16(buf), nil +} + +// getReadBytes returns read bytes +func (b *byteInputAdapter) getReadBytes() int64 { + return int64(b.readBytes) +} + +// skipBytes skips exactly n bytes +func (b *byteInputAdapter) skipBytes(n int) error { + _, err := b.next(n) + + return err +} + +// reset resets the given buffer with a new stream +func (b *byteInputAdapter) reset(stream io.Reader) { + b.r = stream + b.readBytes = 0 +} diff --git a/vendor/github.com/RoaringBitmap/roaring/clz.go b/vendor/github.com/RoaringBitmap/roaring/clz.go new file mode 100644 index 0000000000..bcd80d32f0 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/clz.go @@ -0,0 +1,11 @@ +// +build go1.9 +// "go1.9", from Go version 1.9 onward +// See https://golang.org/pkg/go/build/#hdr-Build_Constraints + +package roaring + +import "math/bits" + +func countLeadingZeros(x uint64) int { + return bits.LeadingZeros64(x) +} diff --git a/vendor/github.com/RoaringBitmap/roaring/clz_compat.go b/vendor/github.com/RoaringBitmap/roaring/clz_compat.go new file mode 100644 index 0000000000..eeef4de35b --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/clz_compat.go @@ -0,0 +1,36 @@ +// +build !go1.9 + +package roaring + +// LeadingZeroBits returns the number of consecutive most significant zero +// bits of x. +func countLeadingZeros(i uint64) int { + if i == 0 { + return 64 + } + n := 1 + x := uint32(i >> 32) + if x == 0 { + n += 32 + x = uint32(i) + } + if (x >> 16) == 0 { + n += 16 + x <<= 16 + } + if (x >> 24) == 0 { + n += 8 + x <<= 8 + } + if x>>28 == 0 { + n += 4 + x <<= 4 + } + if x>>30 == 0 { + n += 2 + x <<= 2 + + } + n -= int(x >> 31) + return n +} diff --git a/vendor/github.com/RoaringBitmap/roaring/go.mod b/vendor/github.com/RoaringBitmap/roaring/go.mod new file mode 100644 index 0000000000..f5aebf3967 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/go.mod @@ -0,0 +1,16 @@ +module github.com/RoaringBitmap/roaring + +go 1.12 + +require ( + github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 + github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 // indirect + github.com/jtolds/gls v4.20.0+incompatible // indirect + github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae + github.com/philhofer/fwd v1.0.0 // indirect + github.com/stretchr/testify v1.4.0 + github.com/tinylib/msgp v1.1.0 + github.com/willf/bitset v1.1.10 +) diff --git a/vendor/github.com/RoaringBitmap/roaring/go.sum b/vendor/github.com/RoaringBitmap/roaring/go.sum new file mode 100644 index 0000000000..2e27dbb6e6 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/go.sum @@ -0,0 +1,30 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 h1:twflg0XRTjwKpxb/jFExr4HGq6on2dEOmnL6FV+fgPw= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= +github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/github.com/RoaringBitmap/roaring/manyiterator.go b/vendor/github.com/RoaringBitmap/roaring/manyiterator.go index b4f630a7b4..3007563775 100644 --- a/vendor/github.com/RoaringBitmap/roaring/manyiterator.go +++ b/vendor/github.com/RoaringBitmap/roaring/manyiterator.go @@ -4,12 +4,7 @@ type manyIterable interface { nextMany(hs uint32, buf []uint32) int } -type manyIterator struct { - slice []uint16 - loc int -} - -func (si *manyIterator) nextMany(hs uint32, buf []uint32) int { +func (si *shortIterator) nextMany(hs uint32, buf []uint32) int { n := 0 l := si.loc s := si.slice diff --git a/vendor/github.com/RoaringBitmap/roaring/parallel.go b/vendor/github.com/RoaringBitmap/roaring/parallel.go index 09f94fe83c..2af1aed48e 100644 --- a/vendor/github.com/RoaringBitmap/roaring/parallel.go +++ b/vendor/github.com/RoaringBitmap/roaring/parallel.go @@ -143,8 +143,8 @@ func toBitmapContainer(c container) container { func appenderRoutine(bitmapChan chan<- *Bitmap, resultChan <-chan keyedContainer, expectedKeysChan <-chan int) { expectedKeys := -1 appendedKeys := 0 - keys := make([]uint16, 0) - containers := make([]container, 0) + var keys []uint16 + var containers []container for appendedKeys != expectedKeys { select { case item := <-resultChan: @@ -337,7 +337,7 @@ func ParAnd(parallelism int, bitmaps ...*Bitmap) *Bitmap { // (if it is set to 0, a default number of workers is chosen) func ParOr(parallelism int, bitmaps ...*Bitmap) *Bitmap { var lKey uint16 = MaxUint16 - var hKey uint16 = 0 + var hKey uint16 bitmapsFiltered := bitmaps[:0] for _, b := range bitmaps { diff --git a/vendor/github.com/RoaringBitmap/roaring/rle.go b/vendor/github.com/RoaringBitmap/roaring/rle.go deleted file mode 100644 index 8f3d4edd68..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/rle.go +++ /dev/null @@ -1,1667 +0,0 @@ -package roaring - -// -// Copyright (c) 2016 by the roaring authors. -// Licensed under the Apache License, Version 2.0. -// -// We derive a few lines of code from the sort.Search -// function in the golang standard library. That function -// is Copyright 2009 The Go Authors, and licensed -// under the following BSD-style license. -/* -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -import ( - "fmt" - "sort" - "unsafe" -) - -//go:generate msgp -unexported - -// runContainer32 does run-length encoding of sets of -// uint32 integers. -type runContainer32 struct { - iv []interval32 - card int64 - - // avoid allocation during search - myOpts searchOptions `msg:"-"` -} - -// interval32 is the internal to runContainer32 -// structure that maintains the individual [Start, last] -// closed intervals. -type interval32 struct { - start uint32 - last uint32 -} - -// runlen returns the count of integers in the interval. -func (iv interval32) runlen() int64 { - return 1 + int64(iv.last) - int64(iv.start) -} - -// String produces a human viewable string of the contents. -func (iv interval32) String() string { - return fmt.Sprintf("[%d, %d]", iv.start, iv.last) -} - -func ivalString32(iv []interval32) string { - var s string - var j int - var p interval32 - for j, p = range iv { - s += fmt.Sprintf("%v:[%d, %d], ", j, p.start, p.last) - } - return s -} - -// String produces a human viewable string of the contents. -func (rc *runContainer32) String() string { - if len(rc.iv) == 0 { - return "runContainer32{}" - } - is := ivalString32(rc.iv) - return `runContainer32{` + is + `}` -} - -// uint32Slice is a sort.Sort convenience method -type uint32Slice []uint32 - -// Len returns the length of p. -func (p uint32Slice) Len() int { return len(p) } - -// Less returns p[i] < p[j] -func (p uint32Slice) Less(i, j int) bool { return p[i] < p[j] } - -// Swap swaps elements i and j. -func (p uint32Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } - -//msgp:ignore addHelper - -// addHelper helps build a runContainer32. -type addHelper32 struct { - runstart uint32 - runlen uint32 - actuallyAdded uint32 - m []interval32 - rc *runContainer32 -} - -func (ah *addHelper32) storeIval(runstart, runlen uint32) { - mi := interval32{start: runstart, last: runstart + runlen} - ah.m = append(ah.m, mi) -} - -func (ah *addHelper32) add(cur, prev uint32, i int) { - if cur == prev+1 { - ah.runlen++ - ah.actuallyAdded++ - } else { - if cur < prev { - panic(fmt.Sprintf("newRunContainer32FromVals sees "+ - "unsorted vals; vals[%v]=cur=%v < prev=%v. Sort your vals"+ - " before calling us with alreadySorted == true.", i, cur, prev)) - } - if cur == prev { - // ignore duplicates - } else { - ah.actuallyAdded++ - ah.storeIval(ah.runstart, ah.runlen) - ah.runstart = cur - ah.runlen = 0 - } - } -} - -// newRunContainerRange makes a new container made of just the specified closed interval [rangestart,rangelast] -func newRunContainer32Range(rangestart uint32, rangelast uint32) *runContainer32 { - rc := &runContainer32{} - rc.iv = append(rc.iv, interval32{start: rangestart, last: rangelast}) - return rc -} - -// newRunContainer32FromVals makes a new container from vals. -// -// For efficiency, vals should be sorted in ascending order. -// Ideally vals should not contain duplicates, but we detect and -// ignore them. If vals is already sorted in ascending order, then -// pass alreadySorted = true. Otherwise, for !alreadySorted, -// we will sort vals before creating a runContainer32 of them. -// We sort the original vals, so this will change what the -// caller sees in vals as a side effect. -func newRunContainer32FromVals(alreadySorted bool, vals ...uint32) *runContainer32 { - // keep this in sync with newRunContainer32FromArray below - - rc := &runContainer32{} - ah := addHelper32{rc: rc} - - if !alreadySorted { - sort.Sort(uint32Slice(vals)) - } - n := len(vals) - var cur, prev uint32 - switch { - case n == 0: - // nothing more - case n == 1: - ah.m = append(ah.m, interval32{start: vals[0], last: vals[0]}) - ah.actuallyAdded++ - default: - ah.runstart = vals[0] - ah.actuallyAdded++ - for i := 1; i < n; i++ { - prev = vals[i-1] - cur = vals[i] - ah.add(cur, prev, i) - } - ah.storeIval(ah.runstart, ah.runlen) - } - rc.iv = ah.m - rc.card = int64(ah.actuallyAdded) - return rc -} - -// newRunContainer32FromBitmapContainer makes a new run container from bc, -// somewhat efficiently. For reference, see the Java -// https://github.com/RoaringBitmap/RoaringBitmap/blob/master/src/main/java/org/roaringbitmap/RunContainer.java#L145-L192 -func newRunContainer32FromBitmapContainer(bc *bitmapContainer) *runContainer32 { - - rc := &runContainer32{} - nbrRuns := bc.numberOfRuns() - if nbrRuns == 0 { - return rc - } - rc.iv = make([]interval32, nbrRuns) - - longCtr := 0 // index of current long in bitmap - curWord := bc.bitmap[0] // its value - runCount := 0 - for { - // potentially multiword advance to first 1 bit - for curWord == 0 && longCtr < len(bc.bitmap)-1 { - longCtr++ - curWord = bc.bitmap[longCtr] - } - - if curWord == 0 { - // wrap up, no more runs - return rc - } - localRunStart := countTrailingZeros(curWord) - runStart := localRunStart + 64*longCtr - // stuff 1s into number's LSBs - curWordWith1s := curWord | (curWord - 1) - - // find the next 0, potentially in a later word - runEnd := 0 - for curWordWith1s == maxWord && longCtr < len(bc.bitmap)-1 { - longCtr++ - curWordWith1s = bc.bitmap[longCtr] - } - - if curWordWith1s == maxWord { - // a final unterminated run of 1s - runEnd = wordSizeInBits + longCtr*64 - rc.iv[runCount].start = uint32(runStart) - rc.iv[runCount].last = uint32(runEnd) - 1 - return rc - } - localRunEnd := countTrailingZeros(^curWordWith1s) - runEnd = localRunEnd + longCtr*64 - rc.iv[runCount].start = uint32(runStart) - rc.iv[runCount].last = uint32(runEnd) - 1 - runCount++ - // now, zero out everything right of runEnd. - curWord = curWordWith1s & (curWordWith1s + 1) - // We've lathered and rinsed, so repeat... - } - -} - -// -// newRunContainer32FromArray populates a new -// runContainer32 from the contents of arr. -// -func newRunContainer32FromArray(arr *arrayContainer) *runContainer32 { - // keep this in sync with newRunContainer32FromVals above - - rc := &runContainer32{} - ah := addHelper32{rc: rc} - - n := arr.getCardinality() - var cur, prev uint32 - switch { - case n == 0: - // nothing more - case n == 1: - ah.m = append(ah.m, interval32{start: uint32(arr.content[0]), last: uint32(arr.content[0])}) - ah.actuallyAdded++ - default: - ah.runstart = uint32(arr.content[0]) - ah.actuallyAdded++ - for i := 1; i < n; i++ { - prev = uint32(arr.content[i-1]) - cur = uint32(arr.content[i]) - ah.add(cur, prev, i) - } - ah.storeIval(ah.runstart, ah.runlen) - } - rc.iv = ah.m - rc.card = int64(ah.actuallyAdded) - return rc -} - -// set adds the integers in vals to the set. Vals -// must be sorted in increasing order; if not, you should set -// alreadySorted to false, and we will sort them in place for you. -// (Be aware of this side effect -- it will affect the callers -// view of vals). -// -// If you have a small number of additions to an already -// big runContainer32, calling Add() may be faster. -func (rc *runContainer32) set(alreadySorted bool, vals ...uint32) { - - rc2 := newRunContainer32FromVals(alreadySorted, vals...) - un := rc.union(rc2) - rc.iv = un.iv - rc.card = 0 -} - -// canMerge returns true if the intervals -// a and b either overlap or they are -// contiguous and so can be merged into -// a single interval. -func canMerge32(a, b interval32) bool { - if int64(a.last)+1 < int64(b.start) { - return false - } - return int64(b.last)+1 >= int64(a.start) -} - -// haveOverlap differs from canMerge in that -// it tells you if the intersection of a -// and b would contain an element (otherwise -// it would be the empty set, and we return -// false). -func haveOverlap32(a, b interval32) bool { - if int64(a.last)+1 <= int64(b.start) { - return false - } - return int64(b.last)+1 > int64(a.start) -} - -// mergeInterval32s joins a and b into a -// new interval, and panics if it cannot. -func mergeInterval32s(a, b interval32) (res interval32) { - if !canMerge32(a, b) { - panic(fmt.Sprintf("cannot merge %#v and %#v", a, b)) - } - if b.start < a.start { - res.start = b.start - } else { - res.start = a.start - } - if b.last > a.last { - res.last = b.last - } else { - res.last = a.last - } - return -} - -// intersectInterval32s returns the intersection -// of a and b. The isEmpty flag will be true if -// a and b were disjoint. -func intersectInterval32s(a, b interval32) (res interval32, isEmpty bool) { - if !haveOverlap32(a, b) { - isEmpty = true - return - } - if b.start > a.start { - res.start = b.start - } else { - res.start = a.start - } - if b.last < a.last { - res.last = b.last - } else { - res.last = a.last - } - return -} - -// union merges two runContainer32s, producing -// a new runContainer32 with the union of rc and b. -func (rc *runContainer32) union(b *runContainer32) *runContainer32 { - - // rc is also known as 'a' here, but golint insisted we - // call it rc for consistency with the rest of the methods. - - var m []interval32 - - alim := int64(len(rc.iv)) - blim := int64(len(b.iv)) - - var na int64 // next from a - var nb int64 // next from b - - // merged holds the current merge output, which might - // get additional merges before being appended to m. - var merged interval32 - var mergedUsed bool // is merged being used at the moment? - - var cura interval32 // currently considering this interval32 from a - var curb interval32 // currently considering this interval32 from b - - pass := 0 - for na < alim && nb < blim { - pass++ - cura = rc.iv[na] - curb = b.iv[nb] - - if mergedUsed { - mergedUpdated := false - if canMerge32(cura, merged) { - merged = mergeInterval32s(cura, merged) - na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1) - mergedUpdated = true - } - if canMerge32(curb, merged) { - merged = mergeInterval32s(curb, merged) - nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1) - mergedUpdated = true - } - if !mergedUpdated { - // we know that merged is disjoint from cura and curb - m = append(m, merged) - mergedUsed = false - } - continue - - } else { - // !mergedUsed - if !canMerge32(cura, curb) { - if cura.start < curb.start { - m = append(m, cura) - na++ - } else { - m = append(m, curb) - nb++ - } - } else { - merged = mergeInterval32s(cura, curb) - mergedUsed = true - na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1) - nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1) - } - } - } - var aDone, bDone bool - if na >= alim { - aDone = true - } - if nb >= blim { - bDone = true - } - // finish by merging anything remaining into merged we can: - if mergedUsed { - if !aDone { - aAdds: - for na < alim { - cura = rc.iv[na] - if canMerge32(cura, merged) { - merged = mergeInterval32s(cura, merged) - na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1) - } else { - break aAdds - } - } - - } - - if !bDone { - bAdds: - for nb < blim { - curb = b.iv[nb] - if canMerge32(curb, merged) { - merged = mergeInterval32s(curb, merged) - nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1) - } else { - break bAdds - } - } - - } - - m = append(m, merged) - } - if na < alim { - m = append(m, rc.iv[na:]...) - } - if nb < blim { - m = append(m, b.iv[nb:]...) - } - - res := &runContainer32{iv: m} - return res -} - -// unionCardinality returns the cardinality of the merger of two runContainer32s, the union of rc and b. -func (rc *runContainer32) unionCardinality(b *runContainer32) uint64 { - - // rc is also known as 'a' here, but golint insisted we - // call it rc for consistency with the rest of the methods. - answer := uint64(0) - - alim := int64(len(rc.iv)) - blim := int64(len(b.iv)) - - var na int64 // next from a - var nb int64 // next from b - - // merged holds the current merge output, which might - // get additional merges before being appended to m. - var merged interval32 - var mergedUsed bool // is merged being used at the moment? - - var cura interval32 // currently considering this interval32 from a - var curb interval32 // currently considering this interval32 from b - - pass := 0 - for na < alim && nb < blim { - pass++ - cura = rc.iv[na] - curb = b.iv[nb] - - if mergedUsed { - mergedUpdated := false - if canMerge32(cura, merged) { - merged = mergeInterval32s(cura, merged) - na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1) - mergedUpdated = true - } - if canMerge32(curb, merged) { - merged = mergeInterval32s(curb, merged) - nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1) - mergedUpdated = true - } - if !mergedUpdated { - // we know that merged is disjoint from cura and curb - //m = append(m, merged) - answer += uint64(merged.last) - uint64(merged.start) + 1 - mergedUsed = false - } - continue - - } else { - // !mergedUsed - if !canMerge32(cura, curb) { - if cura.start < curb.start { - answer += uint64(cura.last) - uint64(cura.start) + 1 - //m = append(m, cura) - na++ - } else { - answer += uint64(curb.last) - uint64(curb.start) + 1 - //m = append(m, curb) - nb++ - } - } else { - merged = mergeInterval32s(cura, curb) - mergedUsed = true - na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1) - nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1) - } - } - } - var aDone, bDone bool - if na >= alim { - aDone = true - } - if nb >= blim { - bDone = true - } - // finish by merging anything remaining into merged we can: - if mergedUsed { - if !aDone { - aAdds: - for na < alim { - cura = rc.iv[na] - if canMerge32(cura, merged) { - merged = mergeInterval32s(cura, merged) - na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1) - } else { - break aAdds - } - } - - } - - if !bDone { - bAdds: - for nb < blim { - curb = b.iv[nb] - if canMerge32(curb, merged) { - merged = mergeInterval32s(curb, merged) - nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1) - } else { - break bAdds - } - } - - } - - //m = append(m, merged) - answer += uint64(merged.last) - uint64(merged.start) + 1 - } - for _, r := range rc.iv[na:] { - answer += uint64(r.last) - uint64(r.start) + 1 - } - for _, r := range b.iv[nb:] { - answer += uint64(r.last) - uint64(r.start) + 1 - } - return answer -} - -// indexOfIntervalAtOrAfter is a helper for union. -func (rc *runContainer32) indexOfIntervalAtOrAfter(key int64, startIndex int64) int64 { - rc.myOpts.startIndex = startIndex - rc.myOpts.endxIndex = 0 - - w, already, _ := rc.search(key, &rc.myOpts) - if already { - return w - } - return w + 1 -} - -// intersect returns a new runContainer32 holding the -// intersection of rc (also known as 'a') and b. -func (rc *runContainer32) intersect(b *runContainer32) *runContainer32 { - - a := rc - numa := int64(len(a.iv)) - numb := int64(len(b.iv)) - res := &runContainer32{} - if numa == 0 || numb == 0 { - return res - } - - if numa == 1 && numb == 1 { - if !haveOverlap32(a.iv[0], b.iv[0]) { - return res - } - } - - var output []interval32 - - var acuri int64 - var bcuri int64 - - astart := int64(a.iv[acuri].start) - bstart := int64(b.iv[bcuri].start) - - var intersection interval32 - var leftoverstart int64 - var isOverlap, isLeftoverA, isLeftoverB bool - var done bool - pass := 0 -toploop: - for acuri < numa && bcuri < numb { - pass++ - - isOverlap, isLeftoverA, isLeftoverB, leftoverstart, intersection = intersectWithLeftover32(astart, int64(a.iv[acuri].last), bstart, int64(b.iv[bcuri].last)) - - if !isOverlap { - switch { - case astart < bstart: - acuri, done = a.findNextIntervalThatIntersectsStartingFrom(acuri+1, bstart) - if done { - break toploop - } - astart = int64(a.iv[acuri].start) - - case astart > bstart: - bcuri, done = b.findNextIntervalThatIntersectsStartingFrom(bcuri+1, astart) - if done { - break toploop - } - bstart = int64(b.iv[bcuri].start) - - //default: - // panic("impossible that astart == bstart, since !isOverlap") - } - - } else { - // isOverlap - output = append(output, intersection) - switch { - case isLeftoverA: - // note that we change astart without advancing acuri, - // since we need to capture any 2ndary intersections with a.iv[acuri] - astart = leftoverstart - bcuri++ - if bcuri >= numb { - break toploop - } - bstart = int64(b.iv[bcuri].start) - case isLeftoverB: - // note that we change bstart without advancing bcuri, - // since we need to capture any 2ndary intersections with b.iv[bcuri] - bstart = leftoverstart - acuri++ - if acuri >= numa { - break toploop - } - astart = int64(a.iv[acuri].start) - default: - // neither had leftover, both completely consumed - // optionally, assert for sanity: - //if a.iv[acuri].endx != b.iv[bcuri].endx { - // panic("huh? should only be possible that endx agree now!") - //} - - // advance to next a interval - acuri++ - if acuri >= numa { - break toploop - } - astart = int64(a.iv[acuri].start) - - // advance to next b interval - bcuri++ - if bcuri >= numb { - break toploop - } - bstart = int64(b.iv[bcuri].start) - } - } - } // end for toploop - - if len(output) == 0 { - return res - } - - res.iv = output - return res -} - -// intersectCardinality returns the cardinality of the -// intersection of rc (also known as 'a') and b. -func (rc *runContainer32) intersectCardinality(b *runContainer32) int64 { - answer := int64(0) - - a := rc - numa := int64(len(a.iv)) - numb := int64(len(b.iv)) - if numa == 0 || numb == 0 { - return 0 - } - - if numa == 1 && numb == 1 { - if !haveOverlap32(a.iv[0], b.iv[0]) { - return 0 - } - } - - var acuri int64 - var bcuri int64 - - astart := int64(a.iv[acuri].start) - bstart := int64(b.iv[bcuri].start) - - var intersection interval32 - var leftoverstart int64 - var isOverlap, isLeftoverA, isLeftoverB bool - var done bool - pass := 0 -toploop: - for acuri < numa && bcuri < numb { - pass++ - - isOverlap, isLeftoverA, isLeftoverB, leftoverstart, intersection = intersectWithLeftover32(astart, int64(a.iv[acuri].last), bstart, int64(b.iv[bcuri].last)) - - if !isOverlap { - switch { - case astart < bstart: - acuri, done = a.findNextIntervalThatIntersectsStartingFrom(acuri+1, bstart) - if done { - break toploop - } - astart = int64(a.iv[acuri].start) - - case astart > bstart: - bcuri, done = b.findNextIntervalThatIntersectsStartingFrom(bcuri+1, astart) - if done { - break toploop - } - bstart = int64(b.iv[bcuri].start) - - //default: - // panic("impossible that astart == bstart, since !isOverlap") - } - - } else { - // isOverlap - answer += int64(intersection.last) - int64(intersection.start) + 1 - switch { - case isLeftoverA: - // note that we change astart without advancing acuri, - // since we need to capture any 2ndary intersections with a.iv[acuri] - astart = leftoverstart - bcuri++ - if bcuri >= numb { - break toploop - } - bstart = int64(b.iv[bcuri].start) - case isLeftoverB: - // note that we change bstart without advancing bcuri, - // since we need to capture any 2ndary intersections with b.iv[bcuri] - bstart = leftoverstart - acuri++ - if acuri >= numa { - break toploop - } - astart = int64(a.iv[acuri].start) - default: - // neither had leftover, both completely consumed - // optionally, assert for sanity: - //if a.iv[acuri].endx != b.iv[bcuri].endx { - // panic("huh? should only be possible that endx agree now!") - //} - - // advance to next a interval - acuri++ - if acuri >= numa { - break toploop - } - astart = int64(a.iv[acuri].start) - - // advance to next b interval - bcuri++ - if bcuri >= numb { - break toploop - } - bstart = int64(b.iv[bcuri].start) - } - } - } // end for toploop - - return answer -} - -// get returns true if key is in the container. -func (rc *runContainer32) contains(key uint32) bool { - _, in, _ := rc.search(int64(key), nil) - return in -} - -// numIntervals returns the count of intervals in the container. -func (rc *runContainer32) numIntervals() int { - return len(rc.iv) -} - -// search returns alreadyPresent to indicate if the -// key is already in one of our interval32s. -// -// If key is alreadyPresent, then whichInterval32 tells -// you where. -// -// If key is not already present, then whichInterval32 is -// set as follows: -// -// a) whichInterval32 == len(rc.iv)-1 if key is beyond our -// last interval32 in rc.iv; -// -// b) whichInterval32 == -1 if key is before our first -// interval32 in rc.iv; -// -// c) whichInterval32 is set to the minimum index of rc.iv -// which comes strictly before the key; -// so rc.iv[whichInterval32].last < key, -// and if whichInterval32+1 exists, then key < rc.iv[whichInterval32+1].start -// (Note that whichInterval32+1 won't exist when -// whichInterval32 is the last interval.) -// -// runContainer32.search always returns whichInterval32 < len(rc.iv). -// -// If not nil, opts can be used to further restrict -// the search space. -// -func (rc *runContainer32) search(key int64, opts *searchOptions) (whichInterval32 int64, alreadyPresent bool, numCompares int) { - n := int64(len(rc.iv)) - if n == 0 { - return -1, false, 0 - } - - startIndex := int64(0) - endxIndex := n - if opts != nil { - startIndex = opts.startIndex - - // let endxIndex == 0 mean no effect - if opts.endxIndex > 0 { - endxIndex = opts.endxIndex - } - } - - // sort.Search returns the smallest index i - // in [0, n) at which f(i) is true, assuming that on the range [0, n), - // f(i) == true implies f(i+1) == true. - // If there is no such index, Search returns n. - - // For correctness, this began as verbatim snippet from - // sort.Search in the Go standard lib. - // We inline our comparison function for speed, and - // annotate with numCompares - // to observe and test that extra bounds are utilized. - i, j := startIndex, endxIndex - for i < j { - h := i + (j-i)/2 // avoid overflow when computing h as the bisector - // i <= h < j - numCompares++ - if !(key < int64(rc.iv[h].start)) { - i = h + 1 - } else { - j = h - } - } - below := i - // end std lib snippet. - - // The above is a simple in-lining and annotation of: - /* below := sort.Search(n, - func(i int) bool { - return key < rc.iv[i].start - }) - */ - whichInterval32 = below - 1 - - if below == n { - // all falses => key is >= start of all interval32s - // ... so does it belong to the last interval32? - if key < int64(rc.iv[n-1].last)+1 { - // yes, it belongs to the last interval32 - alreadyPresent = true - return - } - // no, it is beyond the last interval32. - // leave alreadyPreset = false - return - } - - // INVAR: key is below rc.iv[below] - if below == 0 { - // key is before the first first interval32. - // leave alreadyPresent = false - return - } - - // INVAR: key is >= rc.iv[below-1].start and - // key is < rc.iv[below].start - - // is key in below-1 interval32? - if key >= int64(rc.iv[below-1].start) && key < int64(rc.iv[below-1].last)+1 { - // yes, it is. key is in below-1 interval32. - alreadyPresent = true - return - } - - // INVAR: key >= rc.iv[below-1].endx && key < rc.iv[below].start - // leave alreadyPresent = false - return -} - -// cardinality returns the count of the integers stored in the -// runContainer32. -func (rc *runContainer32) cardinality() int64 { - if len(rc.iv) == 0 { - rc.card = 0 - return 0 - } - if rc.card > 0 { - return rc.card // already cached - } - // have to compute it - var n int64 - for _, p := range rc.iv { - n += p.runlen() - } - rc.card = n // cache it - return n -} - -// AsSlice decompresses the contents into a []uint32 slice. -func (rc *runContainer32) AsSlice() []uint32 { - s := make([]uint32, rc.cardinality()) - j := 0 - for _, p := range rc.iv { - for i := p.start; i <= p.last; i++ { - s[j] = i - j++ - } - } - return s -} - -// newRunContainer32 creates an empty run container. -func newRunContainer32() *runContainer32 { - return &runContainer32{} -} - -// newRunContainer32CopyIv creates a run container, initializing -// with a copy of the supplied iv slice. -// -func newRunContainer32CopyIv(iv []interval32) *runContainer32 { - rc := &runContainer32{ - iv: make([]interval32, len(iv)), - } - copy(rc.iv, iv) - return rc -} - -func (rc *runContainer32) Clone() *runContainer32 { - rc2 := newRunContainer32CopyIv(rc.iv) - return rc2 -} - -// newRunContainer32TakeOwnership returns a new runContainer32 -// backed by the provided iv slice, which we will -// assume exclusive control over from now on. -// -func newRunContainer32TakeOwnership(iv []interval32) *runContainer32 { - rc := &runContainer32{ - iv: iv, - } - return rc -} - -const baseRc32Size = int(unsafe.Sizeof(runContainer32{})) -const perIntervalRc32Size = int(unsafe.Sizeof(interval32{})) - -const baseDiskRc32Size = int(unsafe.Sizeof(uint32(0))) - -// see also runContainer32SerializedSizeInBytes(numRuns int) int - -// getSizeInBytes returns the number of bytes of memory -// required by this runContainer32. -func (rc *runContainer32) getSizeInBytes() int { - return perIntervalRc32Size*len(rc.iv) + baseRc32Size -} - -// runContainer32SerializedSizeInBytes returns the number of bytes of disk -// required to hold numRuns in a runContainer32. -func runContainer32SerializedSizeInBytes(numRuns int) int { - return perIntervalRc32Size*numRuns + baseDiskRc32Size -} - -// Add adds a single value k to the set. -func (rc *runContainer32) Add(k uint32) (wasNew bool) { - // TODO comment from runContainer32.java: - // it might be better and simpler to do return - // toBitmapOrArrayContainer(getCardinality()).add(k) - // but note that some unit tests use this method to build up test - // runcontainers without calling runOptimize - - k64 := int64(k) - - index, present, _ := rc.search(k64, nil) - if present { - return // already there - } - wasNew = true - - // increment card if it is cached already - if rc.card > 0 { - rc.card++ - } - n := int64(len(rc.iv)) - if index == -1 { - // we may need to extend the first run - if n > 0 { - if rc.iv[0].start == k+1 { - rc.iv[0].start = k - return - } - } - // nope, k stands alone, starting the new first interval32. - rc.iv = append([]interval32{{start: k, last: k}}, rc.iv...) - return - } - - // are we off the end? handle both index == n and index == n-1: - if index >= n-1 { - if int64(rc.iv[n-1].last)+1 == k64 { - rc.iv[n-1].last++ - return - } - rc.iv = append(rc.iv, interval32{start: k, last: k}) - return - } - - // INVAR: index and index+1 both exist, and k goes between them. - // - // Now: add k into the middle, - // possibly fusing with index or index+1 interval32 - // and possibly resulting in fusing of two interval32s - // that had a one integer gap. - - left := index - right := index + 1 - - // are we fusing left and right by adding k? - if int64(rc.iv[left].last)+1 == k64 && int64(rc.iv[right].start) == k64+1 { - // fuse into left - rc.iv[left].last = rc.iv[right].last - // remove redundant right - rc.iv = append(rc.iv[:left+1], rc.iv[right+1:]...) - return - } - - // are we an addition to left? - if int64(rc.iv[left].last)+1 == k64 { - // yes - rc.iv[left].last++ - return - } - - // are we an addition to right? - if int64(rc.iv[right].start) == k64+1 { - // yes - rc.iv[right].start = k - return - } - - // k makes a standalone new interval32, inserted in the middle - tail := append([]interval32{{start: k, last: k}}, rc.iv[right:]...) - rc.iv = append(rc.iv[:left+1], tail...) - return -} - -//msgp:ignore runIterator - -// runIterator32 advice: you must call Next() at least once -// before calling Cur(); and you should call HasNext() -// before calling Next() to insure there are contents. -type runIterator32 struct { - rc *runContainer32 - curIndex int64 - curPosInIndex uint32 - curSeq int64 -} - -// newRunIterator32 returns a new empty run container. -func (rc *runContainer32) newRunIterator32() *runIterator32 { - return &runIterator32{rc: rc, curIndex: -1} -} - -// HasNext returns false if calling Next will panic. It -// returns true when there is at least one more value -// available in the iteration sequence. -func (ri *runIterator32) hasNext() bool { - if len(ri.rc.iv) == 0 { - return false - } - if ri.curIndex == -1 { - return true - } - return ri.curSeq+1 < ri.rc.cardinality() -} - -// cur returns the current value pointed to by the iterator. -func (ri *runIterator32) cur() uint32 { - return ri.rc.iv[ri.curIndex].start + ri.curPosInIndex -} - -// Next returns the next value in the iteration sequence. -func (ri *runIterator32) next() uint32 { - if !ri.hasNext() { - panic("no Next available") - } - if ri.curIndex >= int64(len(ri.rc.iv)) { - panic("runIterator.Next() going beyond what is available") - } - if ri.curIndex == -1 { - // first time is special - ri.curIndex = 0 - } else { - ri.curPosInIndex++ - if int64(ri.rc.iv[ri.curIndex].start)+int64(ri.curPosInIndex) == int64(ri.rc.iv[ri.curIndex].last)+1 { - ri.curPosInIndex = 0 - ri.curIndex++ - } - ri.curSeq++ - } - return ri.cur() -} - -// remove removes the element that the iterator -// is on from the run container. You can use -// Cur if you want to double check what is about -// to be deleted. -func (ri *runIterator32) remove() uint32 { - n := ri.rc.cardinality() - if n == 0 { - panic("runIterator.Remove called on empty runContainer32") - } - cur := ri.cur() - - ri.rc.deleteAt(&ri.curIndex, &ri.curPosInIndex, &ri.curSeq) - return cur -} - -// remove removes key from the container. -func (rc *runContainer32) removeKey(key uint32) (wasPresent bool) { - - var index int64 - var curSeq int64 - index, wasPresent, _ = rc.search(int64(key), nil) - if !wasPresent { - return // already removed, nothing to do. - } - pos := key - rc.iv[index].start - rc.deleteAt(&index, &pos, &curSeq) - return -} - -// internal helper functions - -func (rc *runContainer32) deleteAt(curIndex *int64, curPosInIndex *uint32, curSeq *int64) { - rc.card-- - (*curSeq)-- - ci := *curIndex - pos := *curPosInIndex - - // are we first, last, or in the middle of our interval32? - switch { - case pos == 0: - if int64(rc.iv[ci].start) == int64(rc.iv[ci].last) { - // our interval disappears - rc.iv = append(rc.iv[:ci], rc.iv[ci+1:]...) - // curIndex stays the same, since the delete did - // the advance for us. - *curPosInIndex = 0 - } else { - rc.iv[ci].start++ // no longer overflowable - } - case int64(pos) == rc.iv[ci].runlen()-1: - // last - rc.iv[ci].last-- - // our interval32 cannot disappear, else we would have been pos == 0, case first above. - (*curPosInIndex)-- - // if we leave *curIndex alone, then Next() will work properly even after the delete. - default: - //middle - // split into two, adding an interval32 - new0 := interval32{ - start: rc.iv[ci].start, - last: rc.iv[ci].start + *curPosInIndex - 1} - - new1start := int64(rc.iv[ci].start) + int64(*curPosInIndex) + 1 - if new1start > int64(MaxUint32) { - panic("overflow?!?!") - } - new1 := interval32{ - start: uint32(new1start), - last: rc.iv[ci].last} - tail := append([]interval32{new0, new1}, rc.iv[ci+1:]...) - rc.iv = append(rc.iv[:ci], tail...) - // update curIndex and curPosInIndex - (*curIndex)++ - *curPosInIndex = 0 - } - -} - -func have4Overlap32(astart, alast, bstart, blast int64) bool { - if alast+1 <= bstart { - return false - } - return blast+1 > astart -} - -func intersectWithLeftover32(astart, alast, bstart, blast int64) (isOverlap, isLeftoverA, isLeftoverB bool, leftoverstart int64, intersection interval32) { - if !have4Overlap32(astart, alast, bstart, blast) { - return - } - isOverlap = true - - // do the intersection: - if bstart > astart { - intersection.start = uint32(bstart) - } else { - intersection.start = uint32(astart) - } - switch { - case blast < alast: - isLeftoverA = true - leftoverstart = blast + 1 - intersection.last = uint32(blast) - case alast < blast: - isLeftoverB = true - leftoverstart = alast + 1 - intersection.last = uint32(alast) - default: - // alast == blast - intersection.last = uint32(alast) - } - - return -} - -func (rc *runContainer32) findNextIntervalThatIntersectsStartingFrom(startIndex int64, key int64) (index int64, done bool) { - - rc.myOpts.startIndex = startIndex - rc.myOpts.endxIndex = 0 - - w, _, _ := rc.search(key, &rc.myOpts) - // rc.search always returns w < len(rc.iv) - if w < startIndex { - // not found and comes before lower bound startIndex, - // so just use the lower bound. - if startIndex == int64(len(rc.iv)) { - // also this bump up means that we are done - return startIndex, true - } - return startIndex, false - } - - return w, false -} - -func sliceToString32(m []interval32) string { - s := "" - for i := range m { - s += fmt.Sprintf("%v: %s, ", i, m[i]) - } - return s -} - -// selectInt32 returns the j-th value in the container. -// We panic of j is out of bounds. -func (rc *runContainer32) selectInt32(j uint32) int { - n := rc.cardinality() - if int64(j) > n { - panic(fmt.Sprintf("Cannot select %v since Cardinality is %v", j, n)) - } - - var offset int64 - for k := range rc.iv { - nextOffset := offset + rc.iv[k].runlen() + 1 - if nextOffset > int64(j) { - return int(int64(rc.iv[k].start) + (int64(j) - offset)) - } - offset = nextOffset - } - panic(fmt.Sprintf("Cannot select %v since Cardinality is %v", j, n)) -} - -// helper for invert -func (rc *runContainer32) invertlastInterval(origin uint32, lastIdx int) []interval32 { - cur := rc.iv[lastIdx] - if cur.last == MaxUint32 { - if cur.start == origin { - return nil // empty container - } - return []interval32{{start: origin, last: cur.start - 1}} - } - if cur.start == origin { - return []interval32{{start: cur.last + 1, last: MaxUint32}} - } - // invert splits - return []interval32{ - {start: origin, last: cur.start - 1}, - {start: cur.last + 1, last: MaxUint32}, - } -} - -// invert returns a new container (not inplace), that is -// the inversion of rc. For each bit b in rc, the -// returned value has !b -func (rc *runContainer32) invert() *runContainer32 { - ni := len(rc.iv) - var m []interval32 - switch ni { - case 0: - return &runContainer32{iv: []interval32{{0, MaxUint32}}} - case 1: - return &runContainer32{iv: rc.invertlastInterval(0, 0)} - } - var invstart int64 - ult := ni - 1 - for i, cur := range rc.iv { - if i == ult { - // invertlastInteval will add both intervals (b) and (c) in - // diagram below. - m = append(m, rc.invertlastInterval(uint32(invstart), i)...) - break - } - // INVAR: i and cur are not the last interval, there is a next at i+1 - // - // ........[cur.start, cur.last] ...... [next.start, next.last].... - // ^ ^ ^ - // (a) (b) (c) - // - // Now: we add interval (a); but if (a) is empty, for cur.start==0, we skip it. - if cur.start > 0 { - m = append(m, interval32{start: uint32(invstart), last: cur.start - 1}) - } - invstart = int64(cur.last + 1) - } - return &runContainer32{iv: m} -} - -func (iv interval32) equal(b interval32) bool { - if iv.start == b.start { - return iv.last == b.last - } - return false -} - -func (iv interval32) isSuperSetOf(b interval32) bool { - return iv.start <= b.start && b.last <= iv.last -} - -func (iv interval32) subtractInterval(del interval32) (left []interval32, delcount int64) { - isect, isEmpty := intersectInterval32s(iv, del) - - if isEmpty { - return nil, 0 - } - if del.isSuperSetOf(iv) { - return nil, iv.runlen() - } - - switch { - case isect.start > iv.start && isect.last < iv.last: - new0 := interval32{start: iv.start, last: isect.start - 1} - new1 := interval32{start: isect.last + 1, last: iv.last} - return []interval32{new0, new1}, isect.runlen() - case isect.start == iv.start: - return []interval32{{start: isect.last + 1, last: iv.last}}, isect.runlen() - default: - return []interval32{{start: iv.start, last: isect.start - 1}}, isect.runlen() - } -} - -func (rc *runContainer32) isubtract(del interval32) { - origiv := make([]interval32, len(rc.iv)) - copy(origiv, rc.iv) - n := int64(len(rc.iv)) - if n == 0 { - return // already done. - } - - _, isEmpty := intersectInterval32s( - interval32{ - start: rc.iv[0].start, - last: rc.iv[n-1].last, - }, del) - if isEmpty { - return // done - } - // INVAR there is some intersection between rc and del - istart, startAlready, _ := rc.search(int64(del.start), nil) - ilast, lastAlready, _ := rc.search(int64(del.last), nil) - rc.card = -1 - if istart == -1 { - if ilast == n-1 && !lastAlready { - rc.iv = nil - return - } - } - // some intervals will remain - switch { - case startAlready && lastAlready: - res0, _ := rc.iv[istart].subtractInterval(del) - - // would overwrite values in iv b/c res0 can have len 2. so - // write to origiv instead. - lost := 1 + ilast - istart - changeSize := int64(len(res0)) - lost - newSize := int64(len(rc.iv)) + changeSize - - // rc.iv = append(pre, caboose...) - // return - - if ilast != istart { - res1, _ := rc.iv[ilast].subtractInterval(del) - res0 = append(res0, res1...) - changeSize = int64(len(res0)) - lost - newSize = int64(len(rc.iv)) + changeSize - } - switch { - case changeSize < 0: - // shrink - copy(rc.iv[istart+int64(len(res0)):], rc.iv[ilast+1:]) - copy(rc.iv[istart:istart+int64(len(res0))], res0) - rc.iv = rc.iv[:newSize] - return - case changeSize == 0: - // stay the same - copy(rc.iv[istart:istart+int64(len(res0))], res0) - return - default: - // changeSize > 0 is only possible when ilast == istart. - // Hence we now know: changeSize == 1 and len(res0) == 2 - rc.iv = append(rc.iv, interval32{}) - // len(rc.iv) is correct now, no need to rc.iv = rc.iv[:newSize] - - // copy the tail into place - copy(rc.iv[ilast+2:], rc.iv[ilast+1:]) - // copy the new item(s) into place - copy(rc.iv[istart:istart+2], res0) - return - } - - case !startAlready && !lastAlready: - // we get to discard whole intervals - - // from the search() definition: - - // if del.start is not present, then istart is - // set as follows: - // - // a) istart == n-1 if del.start is beyond our - // last interval32 in rc.iv; - // - // b) istart == -1 if del.start is before our first - // interval32 in rc.iv; - // - // c) istart is set to the minimum index of rc.iv - // which comes strictly before the del.start; - // so del.start > rc.iv[istart].last, - // and if istart+1 exists, then del.start < rc.iv[istart+1].startx - - // if del.last is not present, then ilast is - // set as follows: - // - // a) ilast == n-1 if del.last is beyond our - // last interval32 in rc.iv; - // - // b) ilast == -1 if del.last is before our first - // interval32 in rc.iv; - // - // c) ilast is set to the minimum index of rc.iv - // which comes strictly before the del.last; - // so del.last > rc.iv[ilast].last, - // and if ilast+1 exists, then del.last < rc.iv[ilast+1].start - - // INVAR: istart >= 0 - pre := rc.iv[:istart+1] - if ilast == n-1 { - rc.iv = pre - return - } - // INVAR: ilast < n-1 - lost := ilast - istart - changeSize := -lost - newSize := int64(len(rc.iv)) + changeSize - if changeSize != 0 { - copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:]) - } - rc.iv = rc.iv[:newSize] - return - - case startAlready && !lastAlready: - // we can only shrink or stay the same size - // i.e. we either eliminate the whole interval, - // or just cut off the right side. - res0, _ := rc.iv[istart].subtractInterval(del) - if len(res0) > 0 { - // len(res) must be 1 - rc.iv[istart] = res0[0] - } - lost := 1 + (ilast - istart) - changeSize := int64(len(res0)) - lost - newSize := int64(len(rc.iv)) + changeSize - if changeSize != 0 { - copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:]) - } - rc.iv = rc.iv[:newSize] - return - - case !startAlready && lastAlready: - // we can only shrink or stay the same size - res1, _ := rc.iv[ilast].subtractInterval(del) - lost := ilast - istart - changeSize := int64(len(res1)) - lost - newSize := int64(len(rc.iv)) + changeSize - if changeSize != 0 { - // move the tail first to make room for res1 - copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:]) - } - copy(rc.iv[istart+1:], res1) - rc.iv = rc.iv[:newSize] - return - } -} - -// compute rc minus b, and return the result as a new value (not inplace). -// port of run_container_andnot from CRoaring... -// https://github.com/RoaringBitmap/CRoaring/blob/master/src/containers/run.c#L435-L496 -func (rc *runContainer32) AndNotRunContainer32(b *runContainer32) *runContainer32 { - - if len(b.iv) == 0 || len(rc.iv) == 0 { - return rc - } - - dst := newRunContainer32() - apos := 0 - bpos := 0 - - a := rc - - astart := a.iv[apos].start - alast := a.iv[apos].last - bstart := b.iv[bpos].start - blast := b.iv[bpos].last - - alen := len(a.iv) - blen := len(b.iv) - - for apos < alen && bpos < blen { - switch { - case alast < bstart: - // output the first run - dst.iv = append(dst.iv, interval32{start: astart, last: alast}) - apos++ - if apos < alen { - astart = a.iv[apos].start - alast = a.iv[apos].last - } - case blast < astart: - // exit the second run - bpos++ - if bpos < blen { - bstart = b.iv[bpos].start - blast = b.iv[bpos].last - } - default: - // a: [ ] - // b: [ ] - // alast >= bstart - // blast >= astart - if astart < bstart { - dst.iv = append(dst.iv, interval32{start: astart, last: bstart - 1}) - } - if alast > blast { - astart = blast + 1 - } else { - apos++ - if apos < alen { - astart = a.iv[apos].start - alast = a.iv[apos].last - } - } - } - } - if apos < alen { - dst.iv = append(dst.iv, interval32{start: astart, last: alast}) - apos++ - if apos < alen { - dst.iv = append(dst.iv, a.iv[apos:]...) - } - } - - return dst -} - -func (rc *runContainer32) numberOfRuns() (nr int) { - return len(rc.iv) -} - -func (rc *runContainer32) containerType() contype { - return run32Contype -} - -func (rc *runContainer32) equals32(srb *runContainer32) bool { - //p("both rc32") - // Check if the containers are the same object. - if rc == srb { - //p("same object") - return true - } - - if len(srb.iv) != len(rc.iv) { - //p("iv len differ") - return false - } - - for i, v := range rc.iv { - if v != srb.iv[i] { - //p("differ at iv i=%v, srb.iv[i]=%v, rc.iv[i]=%v", i, srb.iv[i], rc.iv[i]) - return false - } - } - //p("all intervals same, returning true") - return true -} diff --git a/vendor/github.com/RoaringBitmap/roaring/rle_gen.go b/vendor/github.com/RoaringBitmap/roaring/rle_gen.go deleted file mode 100644 index bc9da75f3a..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/rle_gen.go +++ /dev/null @@ -1,1118 +0,0 @@ -package roaring - -// NOTE: THIS FILE WAS PRODUCED BY THE -// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) -// DO NOT EDIT - -import "github.com/tinylib/msgp/msgp" - -// DecodeMsg implements msgp.Decodable -func (z *addHelper32) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zbai uint32 - zbai, err = dc.ReadMapHeader() - if err != nil { - return - } - for zbai > 0 { - zbai-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "runstart": - z.runstart, err = dc.ReadUint32() - if err != nil { - return - } - case "runlen": - z.runlen, err = dc.ReadUint32() - if err != nil { - return - } - case "actuallyAdded": - z.actuallyAdded, err = dc.ReadUint32() - if err != nil { - return - } - case "m": - var zcmr uint32 - zcmr, err = dc.ReadArrayHeader() - if err != nil { - return - } - if cap(z.m) >= int(zcmr) { - z.m = (z.m)[:zcmr] - } else { - z.m = make([]interval32, zcmr) - } - for zxvk := range z.m { - var zajw uint32 - zajw, err = dc.ReadMapHeader() - if err != nil { - return - } - for zajw > 0 { - zajw-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "start": - z.m[zxvk].start, err = dc.ReadUint32() - if err != nil { - return - } - case "last": - z.m[zxvk].last, err = dc.ReadUint32() - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - } - case "rc": - if dc.IsNil() { - err = dc.ReadNil() - if err != nil { - return - } - z.rc = nil - } else { - if z.rc == nil { - z.rc = new(runContainer32) - } - var zwht uint32 - zwht, err = dc.ReadMapHeader() - if err != nil { - return - } - for zwht > 0 { - zwht-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "iv": - var zhct uint32 - zhct, err = dc.ReadArrayHeader() - if err != nil { - return - } - if cap(z.rc.iv) >= int(zhct) { - z.rc.iv = (z.rc.iv)[:zhct] - } else { - z.rc.iv = make([]interval32, zhct) - } - for zbzg := range z.rc.iv { - var zcua uint32 - zcua, err = dc.ReadMapHeader() - if err != nil { - return - } - for zcua > 0 { - zcua-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "start": - z.rc.iv[zbzg].start, err = dc.ReadUint32() - if err != nil { - return - } - case "last": - z.rc.iv[zbzg].last, err = dc.ReadUint32() - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - } - case "card": - z.rc.card, err = dc.ReadInt64() - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - return -} - -// EncodeMsg implements msgp.Encodable -func (z *addHelper32) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 5 - // write "runstart" - err = en.Append(0x85, 0xa8, 0x72, 0x75, 0x6e, 0x73, 0x74, 0x61, 0x72, 0x74) - if err != nil { - return err - } - err = en.WriteUint32(z.runstart) - if err != nil { - return - } - // write "runlen" - err = en.Append(0xa6, 0x72, 0x75, 0x6e, 0x6c, 0x65, 0x6e) - if err != nil { - return err - } - err = en.WriteUint32(z.runlen) - if err != nil { - return - } - // write "actuallyAdded" - err = en.Append(0xad, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x6c, 0x79, 0x41, 0x64, 0x64, 0x65, 0x64) - if err != nil { - return err - } - err = en.WriteUint32(z.actuallyAdded) - if err != nil { - return - } - // write "m" - err = en.Append(0xa1, 0x6d) - if err != nil { - return err - } - err = en.WriteArrayHeader(uint32(len(z.m))) - if err != nil { - return - } - for zxvk := range z.m { - // map header, size 2 - // write "start" - err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) - if err != nil { - return err - } - err = en.WriteUint32(z.m[zxvk].start) - if err != nil { - return - } - // write "last" - err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74) - if err != nil { - return err - } - err = en.WriteUint32(z.m[zxvk].last) - if err != nil { - return - } - } - // write "rc" - err = en.Append(0xa2, 0x72, 0x63) - if err != nil { - return err - } - if z.rc == nil { - err = en.WriteNil() - if err != nil { - return - } - } else { - // map header, size 2 - // write "iv" - err = en.Append(0x82, 0xa2, 0x69, 0x76) - if err != nil { - return err - } - err = en.WriteArrayHeader(uint32(len(z.rc.iv))) - if err != nil { - return - } - for zbzg := range z.rc.iv { - // map header, size 2 - // write "start" - err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) - if err != nil { - return err - } - err = en.WriteUint32(z.rc.iv[zbzg].start) - if err != nil { - return - } - // write "last" - err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74) - if err != nil { - return err - } - err = en.WriteUint32(z.rc.iv[zbzg].last) - if err != nil { - return - } - } - // write "card" - err = en.Append(0xa4, 0x63, 0x61, 0x72, 0x64) - if err != nil { - return err - } - err = en.WriteInt64(z.rc.card) - if err != nil { - return - } - } - return -} - -// MarshalMsg implements msgp.Marshaler -func (z *addHelper32) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 5 - // string "runstart" - o = append(o, 0x85, 0xa8, 0x72, 0x75, 0x6e, 0x73, 0x74, 0x61, 0x72, 0x74) - o = msgp.AppendUint32(o, z.runstart) - // string "runlen" - o = append(o, 0xa6, 0x72, 0x75, 0x6e, 0x6c, 0x65, 0x6e) - o = msgp.AppendUint32(o, z.runlen) - // string "actuallyAdded" - o = append(o, 0xad, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x6c, 0x79, 0x41, 0x64, 0x64, 0x65, 0x64) - o = msgp.AppendUint32(o, z.actuallyAdded) - // string "m" - o = append(o, 0xa1, 0x6d) - o = msgp.AppendArrayHeader(o, uint32(len(z.m))) - for zxvk := range z.m { - // map header, size 2 - // string "start" - o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) - o = msgp.AppendUint32(o, z.m[zxvk].start) - // string "last" - o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74) - o = msgp.AppendUint32(o, z.m[zxvk].last) - } - // string "rc" - o = append(o, 0xa2, 0x72, 0x63) - if z.rc == nil { - o = msgp.AppendNil(o) - } else { - // map header, size 2 - // string "iv" - o = append(o, 0x82, 0xa2, 0x69, 0x76) - o = msgp.AppendArrayHeader(o, uint32(len(z.rc.iv))) - for zbzg := range z.rc.iv { - // map header, size 2 - // string "start" - o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) - o = msgp.AppendUint32(o, z.rc.iv[zbzg].start) - // string "last" - o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74) - o = msgp.AppendUint32(o, z.rc.iv[zbzg].last) - } - // string "card" - o = append(o, 0xa4, 0x63, 0x61, 0x72, 0x64) - o = msgp.AppendInt64(o, z.rc.card) - } - return -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *addHelper32) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zxhx uint32 - zxhx, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zxhx > 0 { - zxhx-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "runstart": - z.runstart, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - return - } - case "runlen": - z.runlen, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - return - } - case "actuallyAdded": - z.actuallyAdded, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - return - } - case "m": - var zlqf uint32 - zlqf, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - return - } - if cap(z.m) >= int(zlqf) { - z.m = (z.m)[:zlqf] - } else { - z.m = make([]interval32, zlqf) - } - for zxvk := range z.m { - var zdaf uint32 - zdaf, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zdaf > 0 { - zdaf-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "start": - z.m[zxvk].start, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - return - } - case "last": - z.m[zxvk].last, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - } - case "rc": - if msgp.IsNil(bts) { - bts, err = msgp.ReadNilBytes(bts) - if err != nil { - return - } - z.rc = nil - } else { - if z.rc == nil { - z.rc = new(runContainer32) - } - var zpks uint32 - zpks, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zpks > 0 { - zpks-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "iv": - var zjfb uint32 - zjfb, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - return - } - if cap(z.rc.iv) >= int(zjfb) { - z.rc.iv = (z.rc.iv)[:zjfb] - } else { - z.rc.iv = make([]interval32, zjfb) - } - for zbzg := range z.rc.iv { - var zcxo uint32 - zcxo, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zcxo > 0 { - zcxo-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "start": - z.rc.iv[zbzg].start, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - return - } - case "last": - z.rc.iv[zbzg].last, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - } - case "card": - z.rc.card, bts, err = msgp.ReadInt64Bytes(bts) - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - o = bts - return -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *addHelper32) Msgsize() (s int) { - s = 1 + 9 + msgp.Uint32Size + 7 + msgp.Uint32Size + 14 + msgp.Uint32Size + 2 + msgp.ArrayHeaderSize + (len(z.m) * (12 + msgp.Uint32Size + msgp.Uint32Size)) + 3 - if z.rc == nil { - s += msgp.NilSize - } else { - s += 1 + 3 + msgp.ArrayHeaderSize + (len(z.rc.iv) * (12 + msgp.Uint32Size + msgp.Uint32Size)) + 5 + msgp.Int64Size - } - return -} - -// DecodeMsg implements msgp.Decodable -func (z *interval32) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zeff uint32 - zeff, err = dc.ReadMapHeader() - if err != nil { - return - } - for zeff > 0 { - zeff-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "start": - z.start, err = dc.ReadUint32() - if err != nil { - return - } - case "last": - z.last, err = dc.ReadUint32() - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - return -} - -// EncodeMsg implements msgp.Encodable -func (z interval32) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 2 - // write "start" - err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) - if err != nil { - return err - } - err = en.WriteUint32(z.start) - if err != nil { - return - } - // write "last" - err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74) - if err != nil { - return err - } - err = en.WriteUint32(z.last) - if err != nil { - return - } - return -} - -// MarshalMsg implements msgp.Marshaler -func (z interval32) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 2 - // string "start" - o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) - o = msgp.AppendUint32(o, z.start) - // string "last" - o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74) - o = msgp.AppendUint32(o, z.last) - return -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *interval32) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zrsw uint32 - zrsw, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zrsw > 0 { - zrsw-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "start": - z.start, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - return - } - case "last": - z.last, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - o = bts - return -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z interval32) Msgsize() (s int) { - s = 1 + 6 + msgp.Uint32Size + 5 + msgp.Uint32Size - return -} - -// DecodeMsg implements msgp.Decodable -func (z *runContainer32) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zdnj uint32 - zdnj, err = dc.ReadMapHeader() - if err != nil { - return - } - for zdnj > 0 { - zdnj-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "iv": - var zobc uint32 - zobc, err = dc.ReadArrayHeader() - if err != nil { - return - } - if cap(z.iv) >= int(zobc) { - z.iv = (z.iv)[:zobc] - } else { - z.iv = make([]interval32, zobc) - } - for zxpk := range z.iv { - var zsnv uint32 - zsnv, err = dc.ReadMapHeader() - if err != nil { - return - } - for zsnv > 0 { - zsnv-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "start": - z.iv[zxpk].start, err = dc.ReadUint32() - if err != nil { - return - } - case "last": - z.iv[zxpk].last, err = dc.ReadUint32() - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - } - case "card": - z.card, err = dc.ReadInt64() - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - return -} - -// EncodeMsg implements msgp.Encodable -func (z *runContainer32) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 2 - // write "iv" - err = en.Append(0x82, 0xa2, 0x69, 0x76) - if err != nil { - return err - } - err = en.WriteArrayHeader(uint32(len(z.iv))) - if err != nil { - return - } - for zxpk := range z.iv { - // map header, size 2 - // write "start" - err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) - if err != nil { - return err - } - err = en.WriteUint32(z.iv[zxpk].start) - if err != nil { - return - } - // write "last" - err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74) - if err != nil { - return err - } - err = en.WriteUint32(z.iv[zxpk].last) - if err != nil { - return - } - } - // write "card" - err = en.Append(0xa4, 0x63, 0x61, 0x72, 0x64) - if err != nil { - return err - } - err = en.WriteInt64(z.card) - if err != nil { - return - } - return -} - -// MarshalMsg implements msgp.Marshaler -func (z *runContainer32) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 2 - // string "iv" - o = append(o, 0x82, 0xa2, 0x69, 0x76) - o = msgp.AppendArrayHeader(o, uint32(len(z.iv))) - for zxpk := range z.iv { - // map header, size 2 - // string "start" - o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) - o = msgp.AppendUint32(o, z.iv[zxpk].start) - // string "last" - o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74) - o = msgp.AppendUint32(o, z.iv[zxpk].last) - } - // string "card" - o = append(o, 0xa4, 0x63, 0x61, 0x72, 0x64) - o = msgp.AppendInt64(o, z.card) - return -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *runContainer32) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zkgt uint32 - zkgt, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zkgt > 0 { - zkgt-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "iv": - var zema uint32 - zema, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - return - } - if cap(z.iv) >= int(zema) { - z.iv = (z.iv)[:zema] - } else { - z.iv = make([]interval32, zema) - } - for zxpk := range z.iv { - var zpez uint32 - zpez, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zpez > 0 { - zpez-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "start": - z.iv[zxpk].start, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - return - } - case "last": - z.iv[zxpk].last, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - } - case "card": - z.card, bts, err = msgp.ReadInt64Bytes(bts) - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - o = bts - return -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *runContainer32) Msgsize() (s int) { - s = 1 + 3 + msgp.ArrayHeaderSize + (len(z.iv) * (12 + msgp.Uint32Size + msgp.Uint32Size)) + 5 + msgp.Int64Size - return -} - -// DecodeMsg implements msgp.Decodable -func (z *runIterator32) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zqke uint32 - zqke, err = dc.ReadMapHeader() - if err != nil { - return - } - for zqke > 0 { - zqke-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "rc": - if dc.IsNil() { - err = dc.ReadNil() - if err != nil { - return - } - z.rc = nil - } else { - if z.rc == nil { - z.rc = new(runContainer32) - } - err = z.rc.DecodeMsg(dc) - if err != nil { - return - } - } - case "curIndex": - z.curIndex, err = dc.ReadInt64() - if err != nil { - return - } - case "curPosInIndex": - z.curPosInIndex, err = dc.ReadUint32() - if err != nil { - return - } - case "curSeq": - z.curSeq, err = dc.ReadInt64() - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - return -} - -// EncodeMsg implements msgp.Encodable -func (z *runIterator32) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 4 - // write "rc" - err = en.Append(0x84, 0xa2, 0x72, 0x63) - if err != nil { - return err - } - if z.rc == nil { - err = en.WriteNil() - if err != nil { - return - } - } else { - err = z.rc.EncodeMsg(en) - if err != nil { - return - } - } - // write "curIndex" - err = en.Append(0xa8, 0x63, 0x75, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78) - if err != nil { - return err - } - err = en.WriteInt64(z.curIndex) - if err != nil { - return - } - // write "curPosInIndex" - err = en.Append(0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78) - if err != nil { - return err - } - err = en.WriteUint32(z.curPosInIndex) - if err != nil { - return - } - // write "curSeq" - err = en.Append(0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71) - if err != nil { - return err - } - err = en.WriteInt64(z.curSeq) - if err != nil { - return - } - return -} - -// MarshalMsg implements msgp.Marshaler -func (z *runIterator32) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 4 - // string "rc" - o = append(o, 0x84, 0xa2, 0x72, 0x63) - if z.rc == nil { - o = msgp.AppendNil(o) - } else { - o, err = z.rc.MarshalMsg(o) - if err != nil { - return - } - } - // string "curIndex" - o = append(o, 0xa8, 0x63, 0x75, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78) - o = msgp.AppendInt64(o, z.curIndex) - // string "curPosInIndex" - o = append(o, 0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78) - o = msgp.AppendUint32(o, z.curPosInIndex) - // string "curSeq" - o = append(o, 0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71) - o = msgp.AppendInt64(o, z.curSeq) - return -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *runIterator32) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zqyh uint32 - zqyh, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zqyh > 0 { - zqyh-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "rc": - if msgp.IsNil(bts) { - bts, err = msgp.ReadNilBytes(bts) - if err != nil { - return - } - z.rc = nil - } else { - if z.rc == nil { - z.rc = new(runContainer32) - } - bts, err = z.rc.UnmarshalMsg(bts) - if err != nil { - return - } - } - case "curIndex": - z.curIndex, bts, err = msgp.ReadInt64Bytes(bts) - if err != nil { - return - } - case "curPosInIndex": - z.curPosInIndex, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - return - } - case "curSeq": - z.curSeq, bts, err = msgp.ReadInt64Bytes(bts) - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - o = bts - return -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *runIterator32) Msgsize() (s int) { - s = 1 + 3 - if z.rc == nil { - s += msgp.NilSize - } else { - s += z.rc.Msgsize() - } - s += 9 + msgp.Int64Size + 14 + msgp.Uint32Size + 7 + msgp.Int64Size - return -} - -// DecodeMsg implements msgp.Decodable -func (z *uint32Slice) DecodeMsg(dc *msgp.Reader) (err error) { - var zjpj uint32 - zjpj, err = dc.ReadArrayHeader() - if err != nil { - return - } - if cap((*z)) >= int(zjpj) { - (*z) = (*z)[:zjpj] - } else { - (*z) = make(uint32Slice, zjpj) - } - for zywj := range *z { - (*z)[zywj], err = dc.ReadUint32() - if err != nil { - return - } - } - return -} - -// EncodeMsg implements msgp.Encodable -func (z uint32Slice) EncodeMsg(en *msgp.Writer) (err error) { - err = en.WriteArrayHeader(uint32(len(z))) - if err != nil { - return - } - for zzpf := range z { - err = en.WriteUint32(z[zzpf]) - if err != nil { - return - } - } - return -} - -// MarshalMsg implements msgp.Marshaler -func (z uint32Slice) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - o = msgp.AppendArrayHeader(o, uint32(len(z))) - for zzpf := range z { - o = msgp.AppendUint32(o, z[zzpf]) - } - return -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *uint32Slice) UnmarshalMsg(bts []byte) (o []byte, err error) { - var zgmo uint32 - zgmo, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - return - } - if cap((*z)) >= int(zgmo) { - (*z) = (*z)[:zgmo] - } else { - (*z) = make(uint32Slice, zgmo) - } - for zrfe := range *z { - (*z)[zrfe], bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - return - } - } - o = bts - return -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z uint32Slice) Msgsize() (s int) { - s = msgp.ArrayHeaderSize + (len(z) * (msgp.Uint32Size)) - return -} diff --git a/vendor/github.com/RoaringBitmap/roaring/rlecommon.go b/vendor/github.com/RoaringBitmap/roaring/rlecommon.go deleted file mode 100644 index 133636787a..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/rlecommon.go +++ /dev/null @@ -1,163 +0,0 @@ -package roaring - -import ( - "fmt" -) - -// common to rle32.go and rle16.go - -// rleVerbose controls whether p() prints show up. -// The testing package sets this based on -// testing.Verbose(). -var rleVerbose bool - -// p is a shorthand for fmt.Printf with beginning and -// trailing newlines. p() makes it easy -// to add diagnostic print statements. -func p(format string, args ...interface{}) { - if rleVerbose { - fmt.Printf("\n"+format+"\n", args...) - } -} - -// MaxUint32 is the largest uint32 value. -const MaxUint32 = 4294967295 - -// MaxUint16 is the largest 16 bit unsigned int. -// This is the largest value an interval16 can store. -const MaxUint16 = 65535 - -// searchOptions allows us to accelerate runContainer32.search with -// prior knowledge of (mostly lower) bounds. This is used by Union -// and Intersect. -type searchOptions struct { - // start here instead of at 0 - startIndex int64 - - // upper bound instead of len(rc.iv); - // endxIndex == 0 means ignore the bound and use - // endxIndex == n ==len(rc.iv) which is also - // naturally the default for search() - // when opt = nil. - endxIndex int64 -} - -// And finds the intersection of rc and b. -func (rc *runContainer32) And(b *Bitmap) *Bitmap { - out := NewBitmap() - for _, p := range rc.iv { - for i := p.start; i <= p.last; i++ { - if b.Contains(i) { - out.Add(i) - } - } - } - return out -} - -// Xor returns the exclusive-or of rc and b. -func (rc *runContainer32) Xor(b *Bitmap) *Bitmap { - out := b.Clone() - for _, p := range rc.iv { - for v := p.start; v <= p.last; v++ { - if out.Contains(v) { - out.RemoveRange(uint64(v), uint64(v+1)) - } else { - out.Add(v) - } - } - } - return out -} - -// Or returns the union of rc and b. -func (rc *runContainer32) Or(b *Bitmap) *Bitmap { - out := b.Clone() - for _, p := range rc.iv { - for v := p.start; v <= p.last; v++ { - out.Add(v) - } - } - return out -} - -// trial is used in the randomized testing of runContainers -type trial struct { - n int - percentFill float64 - ntrial int - - // only in the union test - // only subtract test - percentDelete float64 - - // only in 067 randomized operations - // we do this + 1 passes - numRandomOpsPass int - - // allow sampling range control - // only recent tests respect this. - srang *interval16 -} - -// And finds the intersection of rc and b. -func (rc *runContainer16) And(b *Bitmap) *Bitmap { - out := NewBitmap() - for _, p := range rc.iv { - plast := p.last() - for i := p.start; i <= plast; i++ { - if b.Contains(uint32(i)) { - out.Add(uint32(i)) - } - } - } - return out -} - -// Xor returns the exclusive-or of rc and b. -func (rc *runContainer16) Xor(b *Bitmap) *Bitmap { - out := b.Clone() - for _, p := range rc.iv { - plast := p.last() - for v := p.start; v <= plast; v++ { - w := uint32(v) - if out.Contains(w) { - out.RemoveRange(uint64(w), uint64(w+1)) - } else { - out.Add(w) - } - } - } - return out -} - -// Or returns the union of rc and b. -func (rc *runContainer16) Or(b *Bitmap) *Bitmap { - out := b.Clone() - for _, p := range rc.iv { - plast := p.last() - for v := p.start; v <= plast; v++ { - out.Add(uint32(v)) - } - } - return out -} - -//func (rc *runContainer32) and(container) container { -// panic("TODO. not yet implemented") -//} - -// serializedSizeInBytes returns the number of bytes of memory -// required by this runContainer16. This is for the -// Roaring format, as specified https://github.com/RoaringBitmap/RoaringFormatSpec/ -func (rc *runContainer16) serializedSizeInBytes() int { - // number of runs in one uint16, then each run - // needs two more uint16 - return 2 + len(rc.iv)*4 -} - -// serializedSizeInBytes returns the number of bytes of memory -// required by this runContainer32. -func (rc *runContainer32) serializedSizeInBytes() int { - return 4 + len(rc.iv)*8 -} diff --git a/vendor/github.com/RoaringBitmap/roaring/rlei.go b/vendor/github.com/RoaringBitmap/roaring/rlei.go deleted file mode 100644 index a15a017e47..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/rlei.go +++ /dev/null @@ -1,695 +0,0 @@ -package roaring - -/////////////////////////////////////////////////// -// -// container interface methods for runContainer16 -// -/////////////////////////////////////////////////// - -import ( - "fmt" -) - -// compile time verify we meet interface requirements -var _ container = &runContainer16{} - -func (rc *runContainer16) clone() container { - return newRunContainer16CopyIv(rc.iv) -} - -func (rc *runContainer16) minimum() uint16 { - return rc.iv[0].start // assume not empty -} - -func (rc *runContainer16) maximum() uint16 { - return rc.iv[len(rc.iv)-1].last() // assume not empty -} - -func (rc *runContainer16) isFull() bool { - return (len(rc.iv) == 1) && ((rc.iv[0].start == 0) && (rc.iv[0].last() == MaxUint16)) -} - -func (rc *runContainer16) and(a container) container { - if rc.isFull() { - return a.clone() - } - switch c := a.(type) { - case *runContainer16: - return rc.intersect(c) - case *arrayContainer: - return rc.andArray(c) - case *bitmapContainer: - return rc.andBitmapContainer(c) - } - panic("unsupported container type") -} - -func (rc *runContainer16) andCardinality(a container) int { - switch c := a.(type) { - case *runContainer16: - return int(rc.intersectCardinality(c)) - case *arrayContainer: - return rc.andArrayCardinality(c) - case *bitmapContainer: - return rc.andBitmapContainerCardinality(c) - } - panic("unsupported container type") -} - -// andBitmapContainer finds the intersection of rc and b. -func (rc *runContainer16) andBitmapContainer(bc *bitmapContainer) container { - bc2 := newBitmapContainerFromRun(rc) - return bc2.andBitmap(bc) -} - -func (rc *runContainer16) andArrayCardinality(ac *arrayContainer) int { - pos := 0 - answer := 0 - maxpos := ac.getCardinality() - if maxpos == 0 { - return 0 // won't happen in actual code - } - v := ac.content[pos] -mainloop: - for _, p := range rc.iv { - for v < p.start { - pos++ - if pos == maxpos { - break mainloop - } - v = ac.content[pos] - } - for v <= p.last() { - answer++ - pos++ - if pos == maxpos { - break mainloop - } - v = ac.content[pos] - } - } - return answer -} - -func (rc *runContainer16) iand(a container) container { - if rc.isFull() { - return a.clone() - } - switch c := a.(type) { - case *runContainer16: - return rc.inplaceIntersect(c) - case *arrayContainer: - return rc.andArray(c) - case *bitmapContainer: - return rc.iandBitmapContainer(c) - } - panic("unsupported container type") -} - -func (rc *runContainer16) inplaceIntersect(rc2 *runContainer16) container { - // TODO: optimize by doing less allocation, possibly? - - // sect will be new - sect := rc.intersect(rc2) - *rc = *sect - return rc -} - -func (rc *runContainer16) iandBitmapContainer(bc *bitmapContainer) container { - isect := rc.andBitmapContainer(bc) - *rc = *newRunContainer16FromContainer(isect) - return rc -} - -func (rc *runContainer16) andArray(ac *arrayContainer) container { - if len(rc.iv) == 0 { - return newArrayContainer() - } - - acCardinality := ac.getCardinality() - c := newArrayContainerCapacity(acCardinality) - - for rlePos, arrayPos := 0, 0; arrayPos < acCardinality; { - iv := rc.iv[rlePos] - arrayVal := ac.content[arrayPos] - - for iv.last() < arrayVal { - rlePos++ - if rlePos == len(rc.iv) { - return c - } - iv = rc.iv[rlePos] - } - - if iv.start > arrayVal { - arrayPos = advanceUntil(ac.content, arrayPos, len(ac.content), iv.start) - } else { - c.content = append(c.content, arrayVal) - arrayPos++ - } - } - return c -} - -func (rc *runContainer16) andNot(a container) container { - switch c := a.(type) { - case *arrayContainer: - return rc.andNotArray(c) - case *bitmapContainer: - return rc.andNotBitmap(c) - case *runContainer16: - return rc.andNotRunContainer16(c) - } - panic("unsupported container type") -} - -func (rc *runContainer16) fillLeastSignificant16bits(x []uint32, i int, mask uint32) { - k := 0 - var val int64 - for _, p := range rc.iv { - n := p.runlen() - for j := int64(0); j < n; j++ { - val = int64(p.start) + j - x[k+i] = uint32(val) | mask - k++ - } - } -} - -func (rc *runContainer16) getShortIterator() shortIterable { - return rc.newRunIterator16() -} - -func (rc *runContainer16) getManyIterator() manyIterable { - return rc.newManyRunIterator16() -} - -// add the values in the range [firstOfRange, endx). endx -// is still abe to express 2^16 because it is an int not an uint16. -func (rc *runContainer16) iaddRange(firstOfRange, endx int) container { - - if firstOfRange >= endx { - panic(fmt.Sprintf("invalid %v = endx >= firstOfRange", endx)) - } - addme := newRunContainer16TakeOwnership([]interval16{ - { - start: uint16(firstOfRange), - length: uint16(endx - 1 - firstOfRange), - }, - }) - *rc = *rc.union(addme) - return rc -} - -// remove the values in the range [firstOfRange,endx) -func (rc *runContainer16) iremoveRange(firstOfRange, endx int) container { - if firstOfRange >= endx { - panic(fmt.Sprintf("request to iremove empty set [%v, %v),"+ - " nothing to do.", firstOfRange, endx)) - //return rc - } - x := newInterval16Range(uint16(firstOfRange), uint16(endx-1)) - rc.isubtract(x) - return rc -} - -// not flip the values in the range [firstOfRange,endx) -func (rc *runContainer16) not(firstOfRange, endx int) container { - if firstOfRange >= endx { - panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange)) - } - - return rc.Not(firstOfRange, endx) -} - -// Not flips the values in the range [firstOfRange,endx). -// This is not inplace. Only the returned value has the flipped bits. -// -// Currently implemented as (!A intersect B) union (A minus B), -// where A is rc, and B is the supplied [firstOfRange, endx) interval. -// -// TODO(time optimization): convert this to a single pass -// algorithm by copying AndNotRunContainer16() and modifying it. -// Current routine is correct but -// makes 2 more passes through the arrays than should be -// strictly necessary. Measure both ways though--this may not matter. -// -func (rc *runContainer16) Not(firstOfRange, endx int) *runContainer16 { - - if firstOfRange >= endx { - panic(fmt.Sprintf("invalid %v = endx >= firstOfRange == %v", endx, firstOfRange)) - } - - if firstOfRange >= endx { - return rc.Clone() - } - - a := rc - // algo: - // (!A intersect B) union (A minus B) - - nota := a.invert() - - bs := []interval16{newInterval16Range(uint16(firstOfRange), uint16(endx-1))} - b := newRunContainer16TakeOwnership(bs) - - notAintersectB := nota.intersect(b) - - aMinusB := a.AndNotRunContainer16(b) - - rc2 := notAintersectB.union(aMinusB) - return rc2 -} - -// equals is now logical equals; it does not require the -// same underlying container type. -func (rc *runContainer16) equals(o container) bool { - srb, ok := o.(*runContainer16) - - if !ok { - // maybe value instead of pointer - val, valok := o.(*runContainer16) - if valok { - srb = val - ok = true - } - } - if ok { - // Check if the containers are the same object. - if rc == srb { - return true - } - - if len(srb.iv) != len(rc.iv) { - return false - } - - for i, v := range rc.iv { - if v != srb.iv[i] { - return false - } - } - return true - } - - // use generic comparison - if o.getCardinality() != rc.getCardinality() { - return false - } - rit := rc.getShortIterator() - bit := o.getShortIterator() - - //k := 0 - for rit.hasNext() { - if bit.next() != rit.next() { - return false - } - //k++ - } - return true -} - -func (rc *runContainer16) iaddReturnMinimized(x uint16) container { - rc.Add(x) - return rc -} - -func (rc *runContainer16) iadd(x uint16) (wasNew bool) { - return rc.Add(x) -} - -func (rc *runContainer16) iremoveReturnMinimized(x uint16) container { - rc.removeKey(x) - return rc -} - -func (rc *runContainer16) iremove(x uint16) bool { - return rc.removeKey(x) -} - -func (rc *runContainer16) or(a container) container { - if rc.isFull() { - return rc.clone() - } - switch c := a.(type) { - case *runContainer16: - return rc.union(c) - case *arrayContainer: - return rc.orArray(c) - case *bitmapContainer: - return rc.orBitmapContainer(c) - } - panic("unsupported container type") -} - -func (rc *runContainer16) orCardinality(a container) int { - switch c := a.(type) { - case *runContainer16: - return int(rc.unionCardinality(c)) - case *arrayContainer: - return rc.orArrayCardinality(c) - case *bitmapContainer: - return rc.orBitmapContainerCardinality(c) - } - panic("unsupported container type") -} - -// orBitmapContainer finds the union of rc and bc. -func (rc *runContainer16) orBitmapContainer(bc *bitmapContainer) container { - bc2 := newBitmapContainerFromRun(rc) - return bc2.iorBitmap(bc) -} - -func (rc *runContainer16) andBitmapContainerCardinality(bc *bitmapContainer) int { - answer := 0 - for i := range rc.iv { - answer += bc.getCardinalityInRange(uint(rc.iv[i].start), uint(rc.iv[i].last())+1) - } - //bc.computeCardinality() - return answer -} - -func (rc *runContainer16) orBitmapContainerCardinality(bc *bitmapContainer) int { - return rc.getCardinality() + bc.getCardinality() - rc.andBitmapContainerCardinality(bc) -} - -// orArray finds the union of rc and ac. -func (rc *runContainer16) orArray(ac *arrayContainer) container { - bc1 := newBitmapContainerFromRun(rc) - bc2 := ac.toBitmapContainer() - return bc1.orBitmap(bc2) -} - -// orArray finds the union of rc and ac. -func (rc *runContainer16) orArrayCardinality(ac *arrayContainer) int { - return ac.getCardinality() + rc.getCardinality() - rc.andArrayCardinality(ac) -} - -func (rc *runContainer16) ior(a container) container { - if rc.isFull() { - return rc - } - switch c := a.(type) { - case *runContainer16: - return rc.inplaceUnion(c) - case *arrayContainer: - return rc.iorArray(c) - case *bitmapContainer: - return rc.iorBitmapContainer(c) - } - panic("unsupported container type") -} - -func (rc *runContainer16) inplaceUnion(rc2 *runContainer16) container { - p("rc.inplaceUnion with len(rc2.iv)=%v", len(rc2.iv)) - for _, p := range rc2.iv { - last := int64(p.last()) - for i := int64(p.start); i <= last; i++ { - rc.Add(uint16(i)) - } - } - return rc -} - -func (rc *runContainer16) iorBitmapContainer(bc *bitmapContainer) container { - - it := bc.getShortIterator() - for it.hasNext() { - rc.Add(it.next()) - } - return rc -} - -func (rc *runContainer16) iorArray(ac *arrayContainer) container { - it := ac.getShortIterator() - for it.hasNext() { - rc.Add(it.next()) - } - return rc -} - -// lazyIOR is described (not yet implemented) in -// this nice note from @lemire on -// https://github.com/RoaringBitmap/roaring/pull/70#issuecomment-263613737 -// -// Description of lazyOR and lazyIOR from @lemire: -// -// Lazy functions are optional and can be simply -// wrapper around non-lazy functions. -// -// The idea of "laziness" is as follows. It is -// inspired by the concept of lazy evaluation -// you might be familiar with (functional programming -// and all that). So a roaring bitmap is -// such that all its containers are, in some -// sense, chosen to use as little memory as -// possible. This is nice. Also, all bitsets -// are "cardinality aware" so that you can do -// fast rank/select queries, or query the -// cardinality of the whole bitmap... very fast, -// without latency. -// -// However, imagine that you are aggregating 100 -// bitmaps together. So you OR the first two, then OR -// that with the third one and so forth. Clearly, -// intermediate bitmaps don't need to be as -// compressed as possible, right? They can be -// in a "dirty state". You only need the end -// result to be in a nice state... which you -// can achieve by calling repairAfterLazy at the end. -// -// The Java/C code does something special for -// the in-place lazy OR runs. The idea is that -// instead of taking two run containers and -// generating a new one, we actually try to -// do the computation in-place through a -// technique invented by @gssiyankai (pinging him!). -// What you do is you check whether the host -// run container has lots of extra capacity. -// If it does, you move its data at the end of -// the backing array, and then you write -// the answer at the beginning. What this -// trick does is minimize memory allocations. -// -func (rc *runContainer16) lazyIOR(a container) container { - // not lazy at the moment - // TODO: make it lazy - return rc.ior(a) - - /* - switch c := a.(type) { - case *arrayContainer: - return rc.lazyIorArray(c) - case *bitmapContainer: - return rc.lazyIorBitmap(c) - case *runContainer16: - return rc.lazyIorRun16(c) - } - panic("unsupported container type") - */ -} - -// lazyOR is described above in lazyIOR. -func (rc *runContainer16) lazyOR(a container) container { - - // not lazy at the moment - // TODO: make it lazy - return rc.or(a) - - /* - switch c := a.(type) { - case *arrayContainer: - return rc.lazyOrArray(c) - case *bitmapContainer: - return rc.lazyOrBitmap(c) - case *runContainer16: - return rc.lazyOrRunContainer16(c) - } - panic("unsupported container type") - */ -} - -func (rc *runContainer16) intersects(a container) bool { - // TODO: optimize by doing inplace/less allocation, possibly? - isect := rc.and(a) - return isect.getCardinality() > 0 -} - -func (rc *runContainer16) xor(a container) container { - switch c := a.(type) { - case *arrayContainer: - return rc.xorArray(c) - case *bitmapContainer: - return rc.xorBitmap(c) - case *runContainer16: - return rc.xorRunContainer16(c) - } - panic("unsupported container type") -} - -func (rc *runContainer16) iandNot(a container) container { - switch c := a.(type) { - case *arrayContainer: - return rc.iandNotArray(c) - case *bitmapContainer: - return rc.iandNotBitmap(c) - case *runContainer16: - return rc.iandNotRunContainer16(c) - } - panic("unsupported container type") -} - -// flip the values in the range [firstOfRange,endx) -func (rc *runContainer16) inot(firstOfRange, endx int) container { - if firstOfRange >= endx { - panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange)) - } - // TODO: minimize copies, do it all inplace; not() makes a copy. - rc = rc.Not(firstOfRange, endx) - return rc -} - -func (rc *runContainer16) getCardinality() int { - return int(rc.cardinality()) -} - -func (rc *runContainer16) rank(x uint16) int { - n := int64(len(rc.iv)) - xx := int64(x) - w, already, _ := rc.search(xx, nil) - if w < 0 { - return 0 - } - if !already && w == n-1 { - return rc.getCardinality() - } - var rnk int64 - if !already { - for i := int64(0); i <= w; i++ { - rnk += rc.iv[i].runlen() - } - return int(rnk) - } - for i := int64(0); i < w; i++ { - rnk += rc.iv[i].runlen() - } - rnk += int64(x-rc.iv[w].start) + 1 - return int(rnk) -} - -func (rc *runContainer16) selectInt(x uint16) int { - return rc.selectInt16(x) -} - -func (rc *runContainer16) andNotRunContainer16(b *runContainer16) container { - return rc.AndNotRunContainer16(b) -} - -func (rc *runContainer16) andNotArray(ac *arrayContainer) container { - rcb := rc.toBitmapContainer() - acb := ac.toBitmapContainer() - return rcb.andNotBitmap(acb) -} - -func (rc *runContainer16) andNotBitmap(bc *bitmapContainer) container { - rcb := rc.toBitmapContainer() - return rcb.andNotBitmap(bc) -} - -func (rc *runContainer16) toBitmapContainer() *bitmapContainer { - p("run16 toBitmap starting; rc has %v ranges", len(rc.iv)) - bc := newBitmapContainer() - for i := range rc.iv { - bc.iaddRange(int(rc.iv[i].start), int(rc.iv[i].last())+1) - } - bc.computeCardinality() - return bc -} - -func (rc *runContainer16) iandNotRunContainer16(x2 *runContainer16) container { - rcb := rc.toBitmapContainer() - x2b := x2.toBitmapContainer() - rcb.iandNotBitmapSurely(x2b) - // TODO: check size and optimize the return value - // TODO: is inplace modification really required? If not, elide the copy. - rc2 := newRunContainer16FromBitmapContainer(rcb) - *rc = *rc2 - return rc -} - -func (rc *runContainer16) iandNotArray(ac *arrayContainer) container { - rcb := rc.toBitmapContainer() - acb := ac.toBitmapContainer() - rcb.iandNotBitmapSurely(acb) - // TODO: check size and optimize the return value - // TODO: is inplace modification really required? If not, elide the copy. - rc2 := newRunContainer16FromBitmapContainer(rcb) - *rc = *rc2 - return rc -} - -func (rc *runContainer16) iandNotBitmap(bc *bitmapContainer) container { - rcb := rc.toBitmapContainer() - rcb.iandNotBitmapSurely(bc) - // TODO: check size and optimize the return value - // TODO: is inplace modification really required? If not, elide the copy. - rc2 := newRunContainer16FromBitmapContainer(rcb) - *rc = *rc2 - return rc -} - -func (rc *runContainer16) xorRunContainer16(x2 *runContainer16) container { - rcb := rc.toBitmapContainer() - x2b := x2.toBitmapContainer() - return rcb.xorBitmap(x2b) -} - -func (rc *runContainer16) xorArray(ac *arrayContainer) container { - rcb := rc.toBitmapContainer() - acb := ac.toBitmapContainer() - return rcb.xorBitmap(acb) -} - -func (rc *runContainer16) xorBitmap(bc *bitmapContainer) container { - rcb := rc.toBitmapContainer() - return rcb.xorBitmap(bc) -} - -// convert to bitmap or array *if needed* -func (rc *runContainer16) toEfficientContainer() container { - - // runContainer16SerializedSizeInBytes(numRuns) - sizeAsRunContainer := rc.getSizeInBytes() - sizeAsBitmapContainer := bitmapContainerSizeInBytes() - card := int(rc.cardinality()) - sizeAsArrayContainer := arrayContainerSizeInBytes(card) - if sizeAsRunContainer <= minOfInt(sizeAsBitmapContainer, sizeAsArrayContainer) { - return rc - } - if card <= arrayDefaultMaxSize { - return rc.toArrayContainer() - } - bc := newBitmapContainerFromRun(rc) - return bc -} - -func (rc *runContainer16) toArrayContainer() *arrayContainer { - ac := newArrayContainer() - for i := range rc.iv { - ac.iaddRange(int(rc.iv[i].start), int(rc.iv[i].last())+1) - } - return ac -} - -func newRunContainer16FromContainer(c container) *runContainer16 { - - switch x := c.(type) { - case *runContainer16: - return x.Clone() - case *arrayContainer: - return newRunContainer16FromArray(x) - case *bitmapContainer: - return newRunContainer16FromBitmapContainer(x) - } - panic("unsupported container type") -} diff --git a/vendor/github.com/RoaringBitmap/roaring/roaring.go b/vendor/github.com/RoaringBitmap/roaring/roaring.go index 5045a41933..df58cc30b5 100644 --- a/vendor/github.com/RoaringBitmap/roaring/roaring.go +++ b/vendor/github.com/RoaringBitmap/roaring/roaring.go @@ -6,12 +6,12 @@ package roaring import ( - "bufio" "bytes" "encoding/base64" "fmt" "io" "strconv" + "sync" ) // Bitmap represents a compressed bitmap where you can add integers. @@ -52,7 +52,7 @@ func (rb *Bitmap) ToBytes() ([]byte, error) { return rb.highlowcontainer.toBytes() } -// WriteToMsgpack writes a msgpack2/snappy-streaming compressed serialized +// Deprecated: WriteToMsgpack writes a msgpack2/snappy-streaming compressed serialized // version of this bitmap to stream. The format is not // compatible with the WriteTo() format, and is // experimental: it may produce smaller on disk @@ -67,8 +67,14 @@ func (rb *Bitmap) WriteToMsgpack(stream io.Writer) (int64, error) { // The format is compatible with other RoaringBitmap // implementations (Java, C) and is documented here: // https://github.com/RoaringBitmap/RoaringFormatSpec -func (rb *Bitmap) ReadFrom(stream io.Reader) (int64, error) { - return rb.highlowcontainer.readFrom(stream) +func (rb *Bitmap) ReadFrom(reader io.Reader) (p int64, err error) { + stream := byteInputAdapterPool.Get().(*byteInputAdapter) + stream.reset(reader) + + p, err = rb.highlowcontainer.readFrom(stream) + byteInputAdapterPool.Put(stream) + + return } // FromBuffer creates a bitmap from its serialized version stored in buffer @@ -87,10 +93,36 @@ func (rb *Bitmap) ReadFrom(stream io.Reader) (int64, error) { // You should *not* change the copy-on-write status of the resulting // bitmaps (SetCopyOnWrite). // -func (rb *Bitmap) FromBuffer(buf []byte) (int64, error) { - return rb.highlowcontainer.fromBuffer(buf) +// If buf becomes unavailable, then a bitmap created with +// FromBuffer would be effectively broken. Furthermore, any +// bitmap derived from this bitmap (e.g., via Or, And) might +// also be broken. Thus, before making buf unavailable, you should +// call CloneCopyOnWriteContainers on all such bitmaps. +// +func (rb *Bitmap) FromBuffer(buf []byte) (p int64, err error) { + stream := byteBufferPool.Get().(*byteBuffer) + stream.reset(buf) + + p, err = rb.highlowcontainer.readFrom(stream) + byteBufferPool.Put(stream) + + return } +var ( + byteBufferPool = sync.Pool{ + New: func() interface{} { + return &byteBuffer{} + }, + } + + byteInputAdapterPool = sync.Pool{ + New: func() interface{} { + return &byteInputAdapter{} + }, + } +) + // RunOptimize attempts to further compress the runs of consecutive values found in the bitmap func (rb *Bitmap) RunOptimize() { rb.highlowcontainer.runOptimize() @@ -101,7 +133,7 @@ func (rb *Bitmap) HasRunCompression() bool { return rb.highlowcontainer.hasRunCompression() } -// ReadFromMsgpack reads a msgpack2/snappy-streaming serialized +// Deprecated: ReadFromMsgpack reads a msgpack2/snappy-streaming serialized // version of this bitmap from stream. The format is // expected is that written by the WriteToMsgpack() // call; see additional notes there. @@ -110,29 +142,15 @@ func (rb *Bitmap) ReadFromMsgpack(stream io.Reader) (int64, error) { } // MarshalBinary implements the encoding.BinaryMarshaler interface for the bitmap +// (same as ToBytes) func (rb *Bitmap) MarshalBinary() ([]byte, error) { - var buf bytes.Buffer - writer := bufio.NewWriter(&buf) - _, err := rb.WriteTo(writer) - if err != nil { - return nil, err - } - err = writer.Flush() - if err != nil { - return nil, err - } - return buf.Bytes(), nil + return rb.ToBytes() } // UnmarshalBinary implements the encoding.BinaryUnmarshaler interface for the bitmap func (rb *Bitmap) UnmarshalBinary(data []byte) error { - var buf bytes.Buffer - _, err := buf.Write(data) - if err != nil { - return err - } - reader := bufio.NewReader(&buf) - _, err = rb.ReadFrom(reader) + r := bytes.NewReader(data) + _, err := rb.ReadFrom(r) return err } @@ -215,10 +233,20 @@ type IntIterable interface { Next() uint32 } +// IntPeekable allows you to look at the next value without advancing and +// advance as long as the next value is smaller than minval +type IntPeekable interface { + IntIterable + // PeekNext peeks the next value without advancing the iterator + PeekNext() uint32 + // AdvanceIfNeeded advances as long as the next value is smaller than minval + AdvanceIfNeeded(minval uint32) +} + type intIterator struct { pos int hs uint32 - iter shortIterable + iter shortPeekable highlowcontainer *roaringArray } @@ -244,6 +272,30 @@ func (ii *intIterator) Next() uint32 { return x } +// PeekNext peeks the next value without advancing the iterator +func (ii *intIterator) PeekNext() uint32 { + return uint32(ii.iter.peekNext()&maxLowBit) | ii.hs +} + +// AdvanceIfNeeded advances as long as the next value is smaller than minval +func (ii *intIterator) AdvanceIfNeeded(minval uint32) { + to := minval >> 16 + + for ii.HasNext() && (ii.hs>>16) < to { + ii.pos++ + ii.init() + } + + if ii.HasNext() && (ii.hs>>16) == to { + ii.iter.advanceIfNeeded(lowbits(minval)) + + if !ii.iter.hasNext() { + ii.pos++ + ii.init() + } + } +} + func newIntIterator(a *Bitmap) *intIterator { p := new(intIterator) p.pos = 0 @@ -252,6 +304,45 @@ func newIntIterator(a *Bitmap) *intIterator { return p } +type intReverseIterator struct { + pos int + hs uint32 + iter shortIterable + highlowcontainer *roaringArray +} + +// HasNext returns true if there are more integers to iterate over +func (ii *intReverseIterator) HasNext() bool { + return ii.pos >= 0 +} + +func (ii *intReverseIterator) init() { + if ii.pos >= 0 { + ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getReverseIterator() + ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16 + } else { + ii.iter = nil + } +} + +// Next returns the next integer +func (ii *intReverseIterator) Next() uint32 { + x := uint32(ii.iter.next()) | ii.hs + if !ii.iter.hasNext() { + ii.pos = ii.pos - 1 + ii.init() + } + return x +} + +func newIntReverseIterator(a *Bitmap) *intReverseIterator { + p := new(intReverseIterator) + p.highlowcontainer = &a.highlowcontainer + p.pos = a.highlowcontainer.size() - 1 + p.init() + return p +} + // ManyIntIterable allows you to iterate over the values in a Bitmap type ManyIntIterable interface { // pass in a buffer to fill up with values, returns how many values were returned @@ -325,12 +416,20 @@ func (rb *Bitmap) String() string { return buffer.String() } -// Iterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order -func (rb *Bitmap) Iterator() IntIterable { +// Iterator creates a new IntPeekable to iterate over the integers contained in the bitmap, in sorted order; +// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove). +func (rb *Bitmap) Iterator() IntPeekable { return newIntIterator(rb) } -// Iterator creates a new ManyIntIterable to iterate over the integers contained in the bitmap, in sorted order +// ReverseIterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order; +// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove). +func (rb *Bitmap) ReverseIterator() IntIterable { + return newIntReverseIterator(rb) +} + +// ManyIterator creates a new ManyIntIterable to iterate over the integers contained in the bitmap, in sorted order; +// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove). func (rb *Bitmap) ManyIterator() ManyIntIterable { return newManyIntIterator(rb) } @@ -374,6 +473,46 @@ func (rb *Bitmap) Equals(o interface{}) bool { return false } +// AddOffset adds the value 'offset' to each and every value in a bitmap, generating a new bitmap in the process +func AddOffset(x *Bitmap, offset uint32) (answer *Bitmap) { + containerOffset := highbits(offset) + inOffset := lowbits(offset) + if inOffset == 0 { + answer = x.Clone() + for pos := 0; pos < answer.highlowcontainer.size(); pos++ { + key := answer.highlowcontainer.getKeyAtIndex(pos) + key += containerOffset + answer.highlowcontainer.keys[pos] = key + } + } else { + answer = New() + for pos := 0; pos < x.highlowcontainer.size(); pos++ { + key := x.highlowcontainer.getKeyAtIndex(pos) + key += containerOffset + c := x.highlowcontainer.getContainerAtIndex(pos) + offsetted := c.addOffset(inOffset) + if offsetted[0].getCardinality() > 0 { + curSize := answer.highlowcontainer.size() + lastkey := uint16(0) + if curSize > 0 { + lastkey = answer.highlowcontainer.getKeyAtIndex(curSize - 1) + } + if curSize > 0 && lastkey == key { + prev := answer.highlowcontainer.getContainerAtIndex(curSize - 1) + orrseult := prev.ior(offsetted[0]) + answer.highlowcontainer.setContainerAtIndex(curSize-1, orrseult) + } else { + answer.highlowcontainer.appendContainer(key, offsetted[0], false) + } + } + if offsetted[1].getCardinality() > 0 { + answer.highlowcontainer.appendContainer(key+1, offsetted[1], false) + } + } + } + return answer +} + // Add the integer x to the bitmap func (rb *Bitmap) Add(x uint32) { hb := highbits(x) @@ -794,11 +933,6 @@ main: } } -/*func (rb *Bitmap) Or(x2 *Bitmap) { - results := Or(rb, x2) // Todo: could be computed in-place for reduced memory usage - rb.highlowcontainer = results.highlowcontainer -}*/ - // AndNot computes the difference between two bitmaps and stores the result in the current bitmap func (rb *Bitmap) AndNot(x2 *Bitmap) { pos1 := 0 @@ -1086,10 +1220,10 @@ func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) { return } - hbStart := highbits(uint32(rangeStart)) - lbStart := lowbits(uint32(rangeStart)) - hbLast := highbits(uint32(rangeEnd - 1)) - lbLast := lowbits(uint32(rangeEnd - 1)) + hbStart := uint32(highbits(uint32(rangeStart))) + lbStart := uint32(lowbits(uint32(rangeStart))) + hbLast := uint32(highbits(uint32(rangeEnd - 1))) + lbLast := uint32(lowbits(uint32(rangeEnd - 1))) var max uint32 = maxLowBit for hb := hbStart; hb <= hbLast; hb++ { @@ -1102,7 +1236,7 @@ func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) { containerLast = uint32(lbLast) } - i := rb.highlowcontainer.getIndex(hb) + i := rb.highlowcontainer.getIndex(uint16(hb)) if i >= 0 { c := rb.highlowcontainer.getWritableContainerAtIndex(i).inot(int(containerStart), int(containerLast)+1) @@ -1113,7 +1247,7 @@ func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) { } } else { // *think* the range of ones must never be // empty. - rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, rangeOfOnes(int(containerStart), int(containerLast))) + rb.highlowcontainer.insertNewKeyValueAt(-i-1, uint16(hb), rangeOfOnes(int(containerStart), int(containerLast))) } } } @@ -1139,24 +1273,24 @@ func (rb *Bitmap) AddRange(rangeStart, rangeEnd uint64) { lbLast := uint32(lowbits(uint32(rangeEnd - 1))) var max uint32 = maxLowBit - for hb := uint16(hbStart); hb <= uint16(hbLast); hb++ { + for hb := hbStart; hb <= hbLast; hb++ { containerStart := uint32(0) - if hb == uint16(hbStart) { + if hb == hbStart { containerStart = lbStart } containerLast := max - if hb == uint16(hbLast) { + if hb == hbLast { containerLast = lbLast } - i := rb.highlowcontainer.getIndex(hb) + i := rb.highlowcontainer.getIndex(uint16(hb)) if i >= 0 { c := rb.highlowcontainer.getWritableContainerAtIndex(i).iaddRange(int(containerStart), int(containerLast)+1) rb.highlowcontainer.setContainerAtIndex(i, c) } else { // *think* the range of ones must never be // empty. - rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, rangeOfOnes(int(containerStart), int(containerLast))) + rb.highlowcontainer.insertNewKeyValueAt(-i-1, uint16(hb), rangeOfOnes(int(containerStart), int(containerLast))) } } } @@ -1243,13 +1377,13 @@ func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap { } answer := NewBitmap() - hbStart := highbits(uint32(rangeStart)) - lbStart := lowbits(uint32(rangeStart)) - hbLast := highbits(uint32(rangeEnd - 1)) - lbLast := lowbits(uint32(rangeEnd - 1)) + hbStart := uint32(highbits(uint32(rangeStart))) + lbStart := uint32(lowbits(uint32(rangeStart))) + hbLast := uint32(highbits(uint32(rangeEnd - 1))) + lbLast := uint32(lowbits(uint32(rangeEnd - 1))) // copy the containers before the active area - answer.highlowcontainer.appendCopiesUntil(bm.highlowcontainer, hbStart) + answer.highlowcontainer.appendCopiesUntil(bm.highlowcontainer, uint16(hbStart)) var max uint32 = maxLowBit for hb := hbStart; hb <= hbLast; hb++ { @@ -1262,23 +1396,23 @@ func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap { containerLast = uint32(lbLast) } - i := bm.highlowcontainer.getIndex(hb) - j := answer.highlowcontainer.getIndex(hb) + i := bm.highlowcontainer.getIndex(uint16(hb)) + j := answer.highlowcontainer.getIndex(uint16(hb)) if i >= 0 { c := bm.highlowcontainer.getContainerAtIndex(i).not(int(containerStart), int(containerLast)+1) if c.getCardinality() > 0 { - answer.highlowcontainer.insertNewKeyValueAt(-j-1, hb, c) + answer.highlowcontainer.insertNewKeyValueAt(-j-1, uint16(hb), c) } } else { // *think* the range of ones must never be // empty. - answer.highlowcontainer.insertNewKeyValueAt(-j-1, hb, + answer.highlowcontainer.insertNewKeyValueAt(-j-1, uint16(hb), rangeOfOnes(int(containerStart), int(containerLast))) } } // copy the containers after the active area. - answer.highlowcontainer.appendCopiesAfter(bm.highlowcontainer, hbLast) + answer.highlowcontainer.appendCopiesAfter(bm.highlowcontainer, uint16(hbLast)) return answer } @@ -1296,6 +1430,21 @@ func (rb *Bitmap) GetCopyOnWrite() (val bool) { return rb.highlowcontainer.copyOnWrite } +// CloneCopyOnWriteContainers clones all containers which have +// needCopyOnWrite set to true. +// This can be used to make sure it is safe to munmap a []byte +// that the roaring array may still have a reference to, after +// calling FromBuffer. +// More generally this function is useful if you call FromBuffer +// to construct a bitmap with a backing array buf +// and then later discard the buf array. Note that you should call +// CloneCopyOnWriteContainers on all bitmaps that were derived +// from the 'FromBuffer' bitmap since they map have dependencies +// on the buf array as well. +func (rb *Bitmap) CloneCopyOnWriteContainers() { + rb.highlowcontainer.cloneCopyOnWriteContainers() +} + // FlipInt calls Flip after casting the parameters (convenience method) func FlipInt(bm *Bitmap, rangeStart, rangeEnd int) *Bitmap { return Flip(bm, uint64(rangeStart), uint64(rangeEnd)) diff --git a/vendor/github.com/RoaringBitmap/roaring/roaringarray.go b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go index d9659159d6..d9d5edda73 100644 --- a/vendor/github.com/RoaringBitmap/roaring/roaringarray.go +++ b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go @@ -4,16 +4,16 @@ import ( "bytes" "encoding/binary" "fmt" - "io" - "io/ioutil" - snappy "github.com/glycerine/go-unsnap-stream" "github.com/tinylib/msgp/msgp" + "io" ) //go:generate msgp -unexported type container interface { + addOffset(uint16) []container + clone() container and(container) container andCardinality(container) int @@ -37,7 +37,8 @@ type container interface { not(start, final int) container // range is [firstOfRange,lastOfRange) inot(firstOfRange, endx int) container // i stands for inplace, range is [firstOfRange,endx) xor(r container) container - getShortIterator() shortIterable + getShortIterator() shortPeekable + getReverseIterator() shortIterable getManyIterator() manyIterable contains(i uint16) bool maximum() uint16 @@ -61,7 +62,6 @@ type container interface { iremoveRange(start, final int) container // i stands for inplace, range is [firstOfRange,lastOfRange) selectInt(x uint16) int // selectInt returns the xth integer in the container serializedSizeInBytes() int - readFrom(io.Reader) (int, error) writeTo(io.Writer) (int, error) numberOfRuns() int @@ -280,6 +280,18 @@ func (ra *roaringArray) clone() *roaringArray { return &sa } +// clone all containers which have needCopyOnWrite set to true +// This can be used to make sure it is safe to munmap a []byte +// that the roaring array may still have a reference to. +func (ra *roaringArray) cloneCopyOnWriteContainers() { + for i, needCopyOnWrite := range ra.needCopyOnWrite { + if needCopyOnWrite { + ra.containers[i] = ra.containers[i].clone() + ra.needCopyOnWrite[i] = false + } + } +} + // unused function: //func (ra *roaringArray) containsKey(x uint16) bool { // return (ra.binarySearch(0, int64(len(ra.keys)), x) >= 0) @@ -456,8 +468,7 @@ func (ra *roaringArray) serializedSizeInBytes() uint64 { // // spec: https://github.com/RoaringBitmap/RoaringFormatSpec // -func (ra *roaringArray) toBytes() ([]byte, error) { - stream := &bytes.Buffer{} +func (ra *roaringArray) writeTo(w io.Writer) (n int64, err error) { hasRun := ra.hasRunCompression() isRunSizeInBytes := 0 cookieSize := 8 @@ -522,79 +533,77 @@ func (ra *roaringArray) toBytes() ([]byte, error) { } } - _, err := stream.Write(buf[:nw]) + written, err := w.Write(buf[:nw]) if err != nil { - return nil, err + return n, err } - for i, c := range ra.containers { - _ = i - _, err := c.writeTo(stream) + n += int64(written) + + for _, c := range ra.containers { + written, err := c.writeTo(w) if err != nil { - return nil, err + return n, err } + n += int64(written) } - return stream.Bytes(), nil + return n, nil } // // spec: https://github.com/RoaringBitmap/RoaringFormatSpec // -func (ra *roaringArray) writeTo(out io.Writer) (int64, error) { - by, err := ra.toBytes() - if err != nil { - return 0, err - } - n, err := out.Write(by) - if err == nil && n < len(by) { - err = io.ErrShortWrite - } - return int64(n), err +func (ra *roaringArray) toBytes() ([]byte, error) { + var buf bytes.Buffer + _, err := ra.writeTo(&buf) + return buf.Bytes(), err } -func (ra *roaringArray) fromBuffer(buf []byte) (int64, error) { - pos := 0 - if len(buf) < 8 { - return 0, fmt.Errorf("buffer too small, expecting at least 8 bytes, was %d", len(buf)) +func (ra *roaringArray) readFrom(stream byteInput) (int64, error) { + cookie, err := stream.readUInt32() + + if err != nil { + return stream.getReadBytes(), fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err) } - cookie := binary.LittleEndian.Uint32(buf) - pos += 4 - var size uint32 // number of containers - haveRunContainers := false + var size uint32 var isRunBitmap []byte - // cookie header if cookie&0x0000FFFF == serialCookie { - haveRunContainers = true - size = uint32(uint16(cookie>>16) + 1) // number of containers - + size = uint32(uint16(cookie>>16) + 1) // create is-run-container bitmap isRunBitmapSize := (int(size) + 7) / 8 - if pos+isRunBitmapSize > len(buf) { - return 0, fmt.Errorf("malformed bitmap, is-run bitmap overruns buffer at %d", pos+isRunBitmapSize) - } + isRunBitmap, err = stream.next(isRunBitmapSize) - isRunBitmap = buf[pos : pos+isRunBitmapSize] - pos += isRunBitmapSize + if err != nil { + return stream.getReadBytes(), fmt.Errorf("malformed bitmap, failed to read is-run bitmap, got: %s", err) + } } else if cookie == serialCookieNoRunContainer { - size = binary.LittleEndian.Uint32(buf[pos:]) - pos += 4 + size, err = stream.readUInt32() + + if err != nil { + return stream.getReadBytes(), fmt.Errorf("malformed bitmap, failed to read a bitmap size: %s", err) + } } else { - return 0, fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header") + return stream.getReadBytes(), fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header") } + if size > (1 << 16) { - return 0, fmt.Errorf("It is logically impossible to have more than (1<<16) containers.") + return stream.getReadBytes(), fmt.Errorf("it is logically impossible to have more than (1<<16) containers") } + // descriptive header - // keycard - is {key, cardinality} tuple slice - if pos+2*2*int(size) > len(buf) { - return 0, fmt.Errorf("malfomred bitmap, key-cardinality slice overruns buffer at %d", pos+2*2*int(size)) + buf, err := stream.next(2 * 2 * int(size)) + + if err != nil { + return stream.getReadBytes(), fmt.Errorf("failed to read descriptive header: %s", err) } - keycard := byteSliceAsUint16Slice(buf[pos : pos+2*2*int(size)]) - pos += 2 * 2 * int(size) - if !haveRunContainers || size >= noOffsetThreshold { - pos += 4 * int(size) + keycard := byteSliceAsUint16Slice(buf) + + if isRunBitmap == nil || size >= noOffsetThreshold { + if err := stream.skipBytes(int(size) * 4); err != nil { + return stream.getReadBytes(), fmt.Errorf("failed to skip bytes: %s", err) + } } // Allocate slices upfront as number of containers is known @@ -603,11 +612,13 @@ func (ra *roaringArray) fromBuffer(buf []byte) (int64, error) { } else { ra.containers = make([]container, size) } + if cap(ra.keys) >= int(size) { ra.keys = ra.keys[:size] } else { ra.keys = make([]uint16, size) } + if cap(ra.needCopyOnWrite) >= int(size) { ra.needCopyOnWrite = ra.needCopyOnWrite[:size] } else { @@ -615,129 +626,62 @@ func (ra *roaringArray) fromBuffer(buf []byte) (int64, error) { } for i := uint32(0); i < size; i++ { - key := uint16(keycard[2*i]) + key := keycard[2*i] card := int(keycard[2*i+1]) + 1 ra.keys[i] = key ra.needCopyOnWrite[i] = true - if haveRunContainers && isRunBitmap[i/8]&(1<<(i%8)) != 0 { + if isRunBitmap != nil && isRunBitmap[i/8]&(1<<(i%8)) != 0 { // run container - nr := binary.LittleEndian.Uint16(buf[pos:]) - pos += 2 - if pos+int(nr)*4 > len(buf) { - return 0, fmt.Errorf("malformed bitmap, a run container overruns buffer at %d:%d", pos, pos+int(nr)*4) + nr, err := stream.readUInt16() + + if err != nil { + return 0, fmt.Errorf("failed to read runtime container size: %s", err) + } + + buf, err := stream.next(int(nr) * 4) + + if err != nil { + return stream.getReadBytes(), fmt.Errorf("failed to read runtime container content: %s", err) } + nb := runContainer16{ - iv: byteSliceAsInterval16Slice(buf[pos : pos+int(nr)*4]), + iv: byteSliceAsInterval16Slice(buf), card: int64(card), } - pos += int(nr) * 4 + ra.containers[i] = &nb } else if card > arrayDefaultMaxSize { // bitmap container + buf, err := stream.next(arrayDefaultMaxSize * 2) + + if err != nil { + return stream.getReadBytes(), fmt.Errorf("failed to read bitmap container: %s", err) + } + nb := bitmapContainer{ cardinality: card, - bitmap: byteSliceAsUint64Slice(buf[pos : pos+arrayDefaultMaxSize*2]), + bitmap: byteSliceAsUint64Slice(buf), } - pos += arrayDefaultMaxSize * 2 + ra.containers[i] = &nb } else { // array container - nb := arrayContainer{ - byteSliceAsUint16Slice(buf[pos : pos+card*2]), - } - pos += card * 2 - ra.containers[i] = &nb - } - } - - return int64(pos), nil -} + buf, err := stream.next(card * 2) -func (ra *roaringArray) readFrom(stream io.Reader) (int64, error) { - pos := 0 - var cookie uint32 - err := binary.Read(stream, binary.LittleEndian, &cookie) - if err != nil { - return 0, fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err) - } - pos += 4 - var size uint32 - haveRunContainers := false - var isRun *bitmapContainer - if cookie&0x0000FFFF == serialCookie { - haveRunContainers = true - size = uint32(uint16(cookie>>16) + 1) - bytesToRead := (int(size) + 7) / 8 - numwords := (bytesToRead + 7) / 8 - by := make([]byte, bytesToRead, numwords*8) - nr, err := io.ReadFull(stream, by) - if err != nil { - return 8 + int64(nr), fmt.Errorf("error in readFrom: could not read the "+ - "runContainer bit flags of length %v bytes: %v", bytesToRead, err) - } - pos += bytesToRead - by = by[:cap(by)] - isRun = newBitmapContainer() - for i := 0; i < numwords; i++ { - isRun.bitmap[i] = binary.LittleEndian.Uint64(by) - by = by[8:] - } - } else if cookie == serialCookieNoRunContainer { - err = binary.Read(stream, binary.LittleEndian, &size) - if err != nil { - return 0, fmt.Errorf("error in roaringArray.readFrom: when reading size, got: %s", err) - } - pos += 4 - } else { - return 0, fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header") - } - if size > (1 << 16) { - return 0, fmt.Errorf("It is logically impossible to have more than (1<<16) containers.") - } - // descriptive header - keycard := make([]uint16, 2*size, 2*size) - err = binary.Read(stream, binary.LittleEndian, keycard) - if err != nil { - return 0, err - } - pos += 2 * 2 * int(size) - // offset header - if !haveRunContainers || size >= noOffsetThreshold { - io.CopyN(ioutil.Discard, stream, 4*int64(size)) // we never skip ahead so this data can be ignored - pos += 4 * int(size) - } - for i := uint32(0); i < size; i++ { - key := int(keycard[2*i]) - card := int(keycard[2*i+1]) + 1 - if haveRunContainers && isRun.contains(uint16(i)) { - nb := newRunContainer16() - nr, err := nb.readFrom(stream) if err != nil { - return 0, err + return stream.getReadBytes(), fmt.Errorf("failed to read array container: %s", err) } - pos += nr - ra.appendContainer(uint16(key), nb, false) - } else if card > arrayDefaultMaxSize { - nb := newBitmapContainer() - nr, err := nb.readFrom(stream) - if err != nil { - return 0, err - } - nb.cardinality = card - pos += nr - ra.appendContainer(keycard[2*i], nb, false) - } else { - nb := newArrayContainerSize(card) - nr, err := nb.readFrom(stream) - if err != nil { - return 0, err + + nb := arrayContainer{ + byteSliceAsUint16Slice(buf), } - pos += nr - ra.appendContainer(keycard[2*i], nb, false) + + ra.containers[i] = &nb } } - return int64(pos), nil + + return stream.getReadBytes(), nil } func (ra *roaringArray) hasRunCompression() bool { diff --git a/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go b/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go index 99fb0f6972..dcd718756a 100644 --- a/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go +++ b/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go @@ -8,7 +8,7 @@ import ( "github.com/tinylib/msgp/msgp" ) -// DecodeMsg implements msgp.Decodable +// Deprecated: DecodeMsg implements msgp.Decodable func (z *containerSerz) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field @@ -48,7 +48,7 @@ func (z *containerSerz) DecodeMsg(dc *msgp.Reader) (err error) { return } -// EncodeMsg implements msgp.Encodable +// Deprecated: EncodeMsg implements msgp.Encodable func (z *containerSerz) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 2 // write "t" @@ -72,7 +72,7 @@ func (z *containerSerz) EncodeMsg(en *msgp.Writer) (err error) { return } -// MarshalMsg implements msgp.Marshaler +// Deprecated: MarshalMsg implements msgp.Marshaler func (z *containerSerz) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 2 @@ -88,7 +88,7 @@ func (z *containerSerz) MarshalMsg(b []byte) (o []byte, err error) { return } -// UnmarshalMsg implements msgp.Unmarshaler +// Deprecated: UnmarshalMsg implements msgp.Unmarshaler func (z *containerSerz) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field @@ -129,13 +129,13 @@ func (z *containerSerz) UnmarshalMsg(bts []byte) (o []byte, err error) { return } -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *containerSerz) Msgsize() (s int) { s = 1 + 2 + msgp.Uint8Size + 2 + z.r.Msgsize() return } -// DecodeMsg implements msgp.Decodable +// Deprecated: DecodeMsg implements msgp.Decodable func (z *contype) DecodeMsg(dc *msgp.Reader) (err error) { { var zajw uint8 @@ -148,7 +148,7 @@ func (z *contype) DecodeMsg(dc *msgp.Reader) (err error) { return } -// EncodeMsg implements msgp.Encodable +// Deprecated: EncodeMsg implements msgp.Encodable func (z contype) EncodeMsg(en *msgp.Writer) (err error) { err = en.WriteUint8(uint8(z)) if err != nil { @@ -157,14 +157,14 @@ func (z contype) EncodeMsg(en *msgp.Writer) (err error) { return } -// MarshalMsg implements msgp.Marshaler +// Deprecated: MarshalMsg implements msgp.Marshaler func (z contype) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) o = msgp.AppendUint8(o, uint8(z)) return } -// UnmarshalMsg implements msgp.Unmarshaler +// Deprecated: UnmarshalMsg implements msgp.Unmarshaler func (z *contype) UnmarshalMsg(bts []byte) (o []byte, err error) { { var zwht uint8 @@ -178,13 +178,13 @@ func (z *contype) UnmarshalMsg(bts []byte) (o []byte, err error) { return } -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z contype) Msgsize() (s int) { s = msgp.Uint8Size return } -// DecodeMsg implements msgp.Decodable +// Deprecated: DecodeMsg implements msgp.Decodable func (z *roaringArray) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field @@ -295,7 +295,7 @@ func (z *roaringArray) DecodeMsg(dc *msgp.Reader) (err error) { return } -// EncodeMsg implements msgp.Encodable +// Deprecated: EncodeMsg implements msgp.Encodable func (z *roaringArray) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 4 // write "keys" @@ -370,7 +370,7 @@ func (z *roaringArray) EncodeMsg(en *msgp.Writer) (err error) { return } -// MarshalMsg implements msgp.Marshaler +// Deprecated: MarshalMsg implements msgp.Marshaler func (z *roaringArray) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 4 @@ -407,7 +407,7 @@ func (z *roaringArray) MarshalMsg(b []byte) (o []byte, err error) { return } -// UnmarshalMsg implements msgp.Unmarshaler +// Deprecated: UnmarshalMsg implements msgp.Unmarshaler func (z *roaringArray) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field @@ -519,7 +519,7 @@ func (z *roaringArray) UnmarshalMsg(bts []byte) (o []byte, err error) { return } -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *roaringArray) Msgsize() (s int) { s = 1 + 5 + msgp.ArrayHeaderSize + (len(z.keys) * (msgp.Uint16Size)) + 16 + msgp.ArrayHeaderSize + (len(z.needCopyOnWrite) * (msgp.BoolSize)) + 12 + msgp.BoolSize + 8 + msgp.ArrayHeaderSize for zxhx := range z.conserz { diff --git a/vendor/github.com/RoaringBitmap/roaring/rle16.go b/vendor/github.com/RoaringBitmap/roaring/runcontainer.go index 951af65f3f..cbffdaf24d 100644 --- a/vendor/github.com/RoaringBitmap/roaring/rle16.go +++ b/vendor/github.com/RoaringBitmap/roaring/runcontainer.go @@ -853,6 +853,21 @@ func (rc *runContainer16) numIntervals() int { return len(rc.iv) } +// searchOptions allows us to accelerate search with +// prior knowledge of (mostly lower) bounds. This is used by Union +// and Intersect. +type searchOptions struct { + // start here instead of at 0 + startIndex int64 + + // upper bound instead of len(rc.iv); + // endxIndex == 0 means ignore the bound and use + // endxIndex == n ==len(rc.iv) which is also + // naturally the default for search() + // when opt = nil. + endxIndex int64 +} + // search returns alreadyPresent to indicate if the // key is already in one of our interval16s. // @@ -1134,135 +1149,163 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) { //msgp:ignore runIterator -// runIterator16 advice: you must call Next() at least once -// before calling Cur(); and you should call HasNext() -// before calling Next() to insure there are contents. +// runIterator16 advice: you must call hasNext() +// before calling next()/peekNext() to insure there are contents. type runIterator16 struct { rc *runContainer16 curIndex int64 curPosInIndex uint16 - curSeq int64 } // newRunIterator16 returns a new empty run container. func (rc *runContainer16) newRunIterator16() *runIterator16 { - return &runIterator16{rc: rc, curIndex: -1} + return &runIterator16{rc: rc, curIndex: 0, curPosInIndex: 0} } -// HasNext returns false if calling Next will panic. It +// hasNext returns false if calling next will panic. It // returns true when there is at least one more value // available in the iteration sequence. func (ri *runIterator16) hasNext() bool { - if len(ri.rc.iv) == 0 { - return false - } - if ri.curIndex == -1 { - return true + return int64(len(ri.rc.iv)) > ri.curIndex+1 || + (int64(len(ri.rc.iv)) == ri.curIndex+1 && ri.rc.iv[ri.curIndex].length >= ri.curPosInIndex) +} + +// next returns the next value in the iteration sequence. +func (ri *runIterator16) next() uint16 { + next := ri.rc.iv[ri.curIndex].start + ri.curPosInIndex + + if ri.curPosInIndex == ri.rc.iv[ri.curIndex].length { + ri.curPosInIndex = 0 + ri.curIndex++ + } else { + ri.curPosInIndex++ } - return ri.curSeq+1 < ri.rc.cardinality() + + return next } -// cur returns the current value pointed to by the iterator. -func (ri *runIterator16) cur() uint16 { +// peekNext returns the next value in the iteration sequence without advancing the iterator +func (ri *runIterator16) peekNext() uint16 { return ri.rc.iv[ri.curIndex].start + ri.curPosInIndex } -// Next returns the next value in the iteration sequence. -func (ri *runIterator16) next() uint16 { - if !ri.hasNext() { - panic("no Next available") +// advanceIfNeeded advances as long as the next value is smaller than minval +func (ri *runIterator16) advanceIfNeeded(minval uint16) { + if !ri.hasNext() || ri.peekNext() >= minval { + return } - if ri.curIndex >= int64(len(ri.rc.iv)) { - panic("runIterator.Next() going beyond what is available") + + opt := &searchOptions{ + startIndex: ri.curIndex, + endxIndex: int64(len(ri.rc.iv)), } - if ri.curIndex == -1 { - // first time is special - ri.curIndex = 0 + + // interval cannot be -1 because of minval > peekNext + interval, isPresent, _ := ri.rc.search(int64(minval), opt) + + // if the minval is present, set the curPosIndex at the right position + if isPresent { + ri.curIndex = interval + ri.curPosInIndex = minval - ri.rc.iv[ri.curIndex].start } else { - ri.curPosInIndex++ - if int64(ri.rc.iv[ri.curIndex].start)+int64(ri.curPosInIndex) == int64(ri.rc.iv[ri.curIndex].last())+1 { - ri.curPosInIndex = 0 - ri.curIndex++ - } - ri.curSeq++ + // otherwise interval is set to to the minimum index of rc.iv + // which comes strictly before the key, that's why we set the next interval + ri.curIndex = interval + 1 + ri.curPosInIndex = 0 } - return ri.cur() } -// remove removes the element that the iterator -// is on from the run container. You can use -// Cur if you want to double check what is about -// to be deleted. -func (ri *runIterator16) remove() uint16 { - n := ri.rc.cardinality() - if n == 0 { - panic("runIterator.Remove called on empty runContainer16") +// runReverseIterator16 advice: you must call hasNext() +// before calling next() to insure there are contents. +type runReverseIterator16 struct { + rc *runContainer16 + curIndex int64 // index into rc.iv + curPosInIndex uint16 // offset in rc.iv[curIndex] +} + +// newRunReverseIterator16 returns a new empty run iterator. +func (rc *runContainer16) newRunReverseIterator16() *runReverseIterator16 { + index := int64(len(rc.iv)) - 1 + pos := uint16(0) + + if index >= 0 { + pos = rc.iv[index].length } - cur := ri.cur() - ri.rc.deleteAt(&ri.curIndex, &ri.curPosInIndex, &ri.curSeq) - return cur + return &runReverseIterator16{ + rc: rc, + curIndex: index, + curPosInIndex: pos, + } } -type manyRunIterator16 struct { - rc *runContainer16 - curIndex int64 - curPosInIndex uint16 - curSeq int64 +// hasNext returns false if calling next will panic. It +// returns true when there is at least one more value +// available in the iteration sequence. +func (ri *runReverseIterator16) hasNext() bool { + return ri.curIndex > 0 || ri.curIndex == 0 && ri.curPosInIndex >= 0 } -func (rc *runContainer16) newManyRunIterator16() *manyRunIterator16 { - return &manyRunIterator16{rc: rc, curIndex: -1} -} +// next returns the next value in the iteration sequence. +func (ri *runReverseIterator16) next() uint16 { + next := ri.rc.iv[ri.curIndex].start + ri.curPosInIndex -func (ri *manyRunIterator16) hasNext() bool { - if len(ri.rc.iv) == 0 { - return false - } - if ri.curIndex == -1 { - return true + if ri.curPosInIndex > 0 { + ri.curPosInIndex-- + } else { + ri.curIndex-- + + if ri.curIndex >= 0 { + ri.curPosInIndex = ri.rc.iv[ri.curIndex].length + } } - return ri.curSeq+1 < ri.rc.cardinality() + + return next +} + +func (rc *runContainer16) newManyRunIterator16() *runIterator16 { + return rc.newRunIterator16() } // hs are the high bits to include to avoid needing to reiterate over the buffer in NextMany -func (ri *manyRunIterator16) nextMany(hs uint32, buf []uint32) int { +func (ri *runIterator16) nextMany(hs uint32, buf []uint32) int { n := 0 + if !ri.hasNext() { return n } + // start and end are inclusive for n < len(buf) { - if ri.curIndex == -1 || int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex) <= 0 { + moreVals := 0 + + if ri.rc.iv[ri.curIndex].length >= ri.curPosInIndex { + // add as many as you can from this seq + moreVals = minOfInt(int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex)+1, len(buf)-n) + base := uint32(ri.rc.iv[ri.curIndex].start+ri.curPosInIndex) | hs + + // allows BCE + buf2 := buf[n : n+moreVals] + for i := range buf2 { + buf2[i] = base + uint32(i) + } + + // update values + n += moreVals + } + + if moreVals+int(ri.curPosInIndex) > int(ri.rc.iv[ri.curIndex].length) { ri.curPosInIndex = 0 ri.curIndex++ + if ri.curIndex == int64(len(ri.rc.iv)) { break } - buf[n] = uint32(ri.rc.iv[ri.curIndex].start) | hs - if ri.curIndex != 0 { - ri.curSeq += 1 - } - n += 1 - // not strictly necessarily due to len(buf)-n min check, but saves some work - continue - } - // add as many as you can from this seq - moreVals := minOfInt(int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex), len(buf)-n) - - base := uint32(ri.rc.iv[ri.curIndex].start+ri.curPosInIndex+1) | hs - - // allows BCE - buf2 := buf[n : n+moreVals] - for i := range buf2 { - buf2[i] = base + uint32(i) + } else { + ri.curPosInIndex += uint16(moreVals) //moreVals always fits in uint16 } - - // update values - ri.curPosInIndex += uint16(moreVals) //moreVals always fits in uint16 - ri.curSeq += int64(moreVals) - n += moreVals } + return n } @@ -1270,21 +1313,19 @@ func (ri *manyRunIterator16) nextMany(hs uint32, buf []uint32) int { func (rc *runContainer16) removeKey(key uint16) (wasPresent bool) { var index int64 - var curSeq int64 index, wasPresent, _ = rc.search(int64(key), nil) if !wasPresent { return // already removed, nothing to do. } pos := key - rc.iv[index].start - rc.deleteAt(&index, &pos, &curSeq) + rc.deleteAt(&index, &pos) return } // internal helper functions -func (rc *runContainer16) deleteAt(curIndex *int64, curPosInIndex *uint16, curSeq *int64) { +func (rc *runContainer16) deleteAt(curIndex *int64, curPosInIndex *uint16) { rc.card-- - *curSeq-- ci := *curIndex pos := *curPosInIndex @@ -1401,7 +1442,7 @@ func (rc *runContainer16) selectInt16(j uint16) int { var offset int64 for k := range rc.iv { - nextOffset := offset + rc.iv[k].runlen() + 1 + nextOffset := offset + rc.iv[k].runlen() if nextOffset > int64(j) { return int(int64(rc.iv[k].start) + (int64(j) - offset)) } @@ -1724,24 +1765,750 @@ func (rc *runContainer16) containerType() contype { } func (rc *runContainer16) equals16(srb *runContainer16) bool { - //p("both rc16") // Check if the containers are the same object. if rc == srb { - //p("same object") return true } if len(srb.iv) != len(rc.iv) { - //p("iv len differ") return false } for i, v := range rc.iv { if v != srb.iv[i] { - //p("differ at iv i=%v, srb.iv[i]=%v, rc.iv[i]=%v", i, srb.iv[i], rc.iv[i]) return false } } - //p("all intervals same, returning true") return true } + +// compile time verify we meet interface requirements +var _ container = &runContainer16{} + +func (rc *runContainer16) clone() container { + return newRunContainer16CopyIv(rc.iv) +} + +func (rc *runContainer16) minimum() uint16 { + return rc.iv[0].start // assume not empty +} + +func (rc *runContainer16) maximum() uint16 { + return rc.iv[len(rc.iv)-1].last() // assume not empty +} + +func (rc *runContainer16) isFull() bool { + return (len(rc.iv) == 1) && ((rc.iv[0].start == 0) && (rc.iv[0].last() == MaxUint16)) +} + +func (rc *runContainer16) and(a container) container { + if rc.isFull() { + return a.clone() + } + switch c := a.(type) { + case *runContainer16: + return rc.intersect(c) + case *arrayContainer: + return rc.andArray(c) + case *bitmapContainer: + return rc.andBitmapContainer(c) + } + panic("unsupported container type") +} + +func (rc *runContainer16) andCardinality(a container) int { + switch c := a.(type) { + case *runContainer16: + return int(rc.intersectCardinality(c)) + case *arrayContainer: + return rc.andArrayCardinality(c) + case *bitmapContainer: + return rc.andBitmapContainerCardinality(c) + } + panic("unsupported container type") +} + +// andBitmapContainer finds the intersection of rc and b. +func (rc *runContainer16) andBitmapContainer(bc *bitmapContainer) container { + bc2 := newBitmapContainerFromRun(rc) + return bc2.andBitmap(bc) +} + +func (rc *runContainer16) andArrayCardinality(ac *arrayContainer) int { + pos := 0 + answer := 0 + maxpos := ac.getCardinality() + if maxpos == 0 { + return 0 // won't happen in actual code + } + v := ac.content[pos] +mainloop: + for _, p := range rc.iv { + for v < p.start { + pos++ + if pos == maxpos { + break mainloop + } + v = ac.content[pos] + } + for v <= p.last() { + answer++ + pos++ + if pos == maxpos { + break mainloop + } + v = ac.content[pos] + } + } + return answer +} + +func (rc *runContainer16) iand(a container) container { + if rc.isFull() { + return a.clone() + } + switch c := a.(type) { + case *runContainer16: + return rc.inplaceIntersect(c) + case *arrayContainer: + return rc.andArray(c) + case *bitmapContainer: + return rc.iandBitmapContainer(c) + } + panic("unsupported container type") +} + +func (rc *runContainer16) inplaceIntersect(rc2 *runContainer16) container { + // TODO: optimize by doing less allocation, possibly? + // sect will be new + sect := rc.intersect(rc2) + *rc = *sect + return rc +} + +func (rc *runContainer16) iandBitmapContainer(bc *bitmapContainer) container { + isect := rc.andBitmapContainer(bc) + *rc = *newRunContainer16FromContainer(isect) + return rc +} + +func (rc *runContainer16) andArray(ac *arrayContainer) container { + if len(rc.iv) == 0 { + return newArrayContainer() + } + + acCardinality := ac.getCardinality() + c := newArrayContainerCapacity(acCardinality) + + for rlePos, arrayPos := 0, 0; arrayPos < acCardinality; { + iv := rc.iv[rlePos] + arrayVal := ac.content[arrayPos] + + for iv.last() < arrayVal { + rlePos++ + if rlePos == len(rc.iv) { + return c + } + iv = rc.iv[rlePos] + } + + if iv.start > arrayVal { + arrayPos = advanceUntil(ac.content, arrayPos, len(ac.content), iv.start) + } else { + c.content = append(c.content, arrayVal) + arrayPos++ + } + } + return c +} + +func (rc *runContainer16) andNot(a container) container { + switch c := a.(type) { + case *arrayContainer: + return rc.andNotArray(c) + case *bitmapContainer: + return rc.andNotBitmap(c) + case *runContainer16: + return rc.andNotRunContainer16(c) + } + panic("unsupported container type") +} + +func (rc *runContainer16) fillLeastSignificant16bits(x []uint32, i int, mask uint32) { + k := 0 + var val int64 + for _, p := range rc.iv { + n := p.runlen() + for j := int64(0); j < n; j++ { + val = int64(p.start) + j + x[k+i] = uint32(val) | mask + k++ + } + } +} + +func (rc *runContainer16) getShortIterator() shortPeekable { + return rc.newRunIterator16() +} + +func (rc *runContainer16) getReverseIterator() shortIterable { + return rc.newRunReverseIterator16() +} + +func (rc *runContainer16) getManyIterator() manyIterable { + return rc.newManyRunIterator16() +} + +// add the values in the range [firstOfRange, endx). endx +// is still abe to express 2^16 because it is an int not an uint16. +func (rc *runContainer16) iaddRange(firstOfRange, endx int) container { + + if firstOfRange >= endx { + panic(fmt.Sprintf("invalid %v = endx >= firstOfRange", endx)) + } + addme := newRunContainer16TakeOwnership([]interval16{ + { + start: uint16(firstOfRange), + length: uint16(endx - 1 - firstOfRange), + }, + }) + *rc = *rc.union(addme) + return rc +} + +// remove the values in the range [firstOfRange,endx) +func (rc *runContainer16) iremoveRange(firstOfRange, endx int) container { + if firstOfRange >= endx { + panic(fmt.Sprintf("request to iremove empty set [%v, %v),"+ + " nothing to do.", firstOfRange, endx)) + //return rc + } + x := newInterval16Range(uint16(firstOfRange), uint16(endx-1)) + rc.isubtract(x) + return rc +} + +// not flip the values in the range [firstOfRange,endx) +func (rc *runContainer16) not(firstOfRange, endx int) container { + if firstOfRange >= endx { + panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange)) + } + + return rc.Not(firstOfRange, endx) +} + +// Not flips the values in the range [firstOfRange,endx). +// This is not inplace. Only the returned value has the flipped bits. +// +// Currently implemented as (!A intersect B) union (A minus B), +// where A is rc, and B is the supplied [firstOfRange, endx) interval. +// +// TODO(time optimization): convert this to a single pass +// algorithm by copying AndNotRunContainer16() and modifying it. +// Current routine is correct but +// makes 2 more passes through the arrays than should be +// strictly necessary. Measure both ways though--this may not matter. +// +func (rc *runContainer16) Not(firstOfRange, endx int) *runContainer16 { + + if firstOfRange >= endx { + panic(fmt.Sprintf("invalid %v = endx >= firstOfRange == %v", endx, firstOfRange)) + } + + if firstOfRange >= endx { + return rc.Clone() + } + + a := rc + // algo: + // (!A intersect B) union (A minus B) + + nota := a.invert() + + bs := []interval16{newInterval16Range(uint16(firstOfRange), uint16(endx-1))} + b := newRunContainer16TakeOwnership(bs) + + notAintersectB := nota.intersect(b) + + aMinusB := a.AndNotRunContainer16(b) + + rc2 := notAintersectB.union(aMinusB) + return rc2 +} + +// equals is now logical equals; it does not require the +// same underlying container type. +func (rc *runContainer16) equals(o container) bool { + srb, ok := o.(*runContainer16) + + if !ok { + // maybe value instead of pointer + val, valok := o.(*runContainer16) + if valok { + srb = val + ok = true + } + } + if ok { + // Check if the containers are the same object. + if rc == srb { + return true + } + + if len(srb.iv) != len(rc.iv) { + return false + } + + for i, v := range rc.iv { + if v != srb.iv[i] { + return false + } + } + return true + } + + // use generic comparison + if o.getCardinality() != rc.getCardinality() { + return false + } + rit := rc.getShortIterator() + bit := o.getShortIterator() + + //k := 0 + for rit.hasNext() { + if bit.next() != rit.next() { + return false + } + //k++ + } + return true +} + +func (rc *runContainer16) iaddReturnMinimized(x uint16) container { + rc.Add(x) + return rc +} + +func (rc *runContainer16) iadd(x uint16) (wasNew bool) { + return rc.Add(x) +} + +func (rc *runContainer16) iremoveReturnMinimized(x uint16) container { + rc.removeKey(x) + return rc +} + +func (rc *runContainer16) iremove(x uint16) bool { + return rc.removeKey(x) +} + +func (rc *runContainer16) or(a container) container { + if rc.isFull() { + return rc.clone() + } + switch c := a.(type) { + case *runContainer16: + return rc.union(c) + case *arrayContainer: + return rc.orArray(c) + case *bitmapContainer: + return rc.orBitmapContainer(c) + } + panic("unsupported container type") +} + +func (rc *runContainer16) orCardinality(a container) int { + switch c := a.(type) { + case *runContainer16: + return int(rc.unionCardinality(c)) + case *arrayContainer: + return rc.orArrayCardinality(c) + case *bitmapContainer: + return rc.orBitmapContainerCardinality(c) + } + panic("unsupported container type") +} + +// orBitmapContainer finds the union of rc and bc. +func (rc *runContainer16) orBitmapContainer(bc *bitmapContainer) container { + bc2 := newBitmapContainerFromRun(rc) + return bc2.iorBitmap(bc) +} + +func (rc *runContainer16) andBitmapContainerCardinality(bc *bitmapContainer) int { + answer := 0 + for i := range rc.iv { + answer += bc.getCardinalityInRange(uint(rc.iv[i].start), uint(rc.iv[i].last())+1) + } + //bc.computeCardinality() + return answer +} + +func (rc *runContainer16) orBitmapContainerCardinality(bc *bitmapContainer) int { + return rc.getCardinality() + bc.getCardinality() - rc.andBitmapContainerCardinality(bc) +} + +// orArray finds the union of rc and ac. +func (rc *runContainer16) orArray(ac *arrayContainer) container { + bc1 := newBitmapContainerFromRun(rc) + bc2 := ac.toBitmapContainer() + return bc1.orBitmap(bc2) +} + +// orArray finds the union of rc and ac. +func (rc *runContainer16) orArrayCardinality(ac *arrayContainer) int { + return ac.getCardinality() + rc.getCardinality() - rc.andArrayCardinality(ac) +} + +func (rc *runContainer16) ior(a container) container { + if rc.isFull() { + return rc + } + switch c := a.(type) { + case *runContainer16: + return rc.inplaceUnion(c) + case *arrayContainer: + return rc.iorArray(c) + case *bitmapContainer: + return rc.iorBitmapContainer(c) + } + panic("unsupported container type") +} + +func (rc *runContainer16) inplaceUnion(rc2 *runContainer16) container { + for _, p := range rc2.iv { + last := int64(p.last()) + for i := int64(p.start); i <= last; i++ { + rc.Add(uint16(i)) + } + } + return rc +} + +func (rc *runContainer16) iorBitmapContainer(bc *bitmapContainer) container { + + it := bc.getShortIterator() + for it.hasNext() { + rc.Add(it.next()) + } + return rc +} + +func (rc *runContainer16) iorArray(ac *arrayContainer) container { + it := ac.getShortIterator() + for it.hasNext() { + rc.Add(it.next()) + } + return rc +} + +// lazyIOR is described (not yet implemented) in +// this nice note from @lemire on +// https://github.com/RoaringBitmap/roaring/pull/70#issuecomment-263613737 +// +// Description of lazyOR and lazyIOR from @lemire: +// +// Lazy functions are optional and can be simply +// wrapper around non-lazy functions. +// +// The idea of "laziness" is as follows. It is +// inspired by the concept of lazy evaluation +// you might be familiar with (functional programming +// and all that). So a roaring bitmap is +// such that all its containers are, in some +// sense, chosen to use as little memory as +// possible. This is nice. Also, all bitsets +// are "cardinality aware" so that you can do +// fast rank/select queries, or query the +// cardinality of the whole bitmap... very fast, +// without latency. +// +// However, imagine that you are aggregating 100 +// bitmaps together. So you OR the first two, then OR +// that with the third one and so forth. Clearly, +// intermediate bitmaps don't need to be as +// compressed as possible, right? They can be +// in a "dirty state". You only need the end +// result to be in a nice state... which you +// can achieve by calling repairAfterLazy at the end. +// +// The Java/C code does something special for +// the in-place lazy OR runs. The idea is that +// instead of taking two run containers and +// generating a new one, we actually try to +// do the computation in-place through a +// technique invented by @gssiyankai (pinging him!). +// What you do is you check whether the host +// run container has lots of extra capacity. +// If it does, you move its data at the end of +// the backing array, and then you write +// the answer at the beginning. What this +// trick does is minimize memory allocations. +// +func (rc *runContainer16) lazyIOR(a container) container { + // not lazy at the moment + return rc.ior(a) +} + +// lazyOR is described above in lazyIOR. +func (rc *runContainer16) lazyOR(a container) container { + // not lazy at the moment + return rc.or(a) +} + +func (rc *runContainer16) intersects(a container) bool { + // TODO: optimize by doing inplace/less allocation, possibly? + isect := rc.and(a) + return isect.getCardinality() > 0 +} + +func (rc *runContainer16) xor(a container) container { + switch c := a.(type) { + case *arrayContainer: + return rc.xorArray(c) + case *bitmapContainer: + return rc.xorBitmap(c) + case *runContainer16: + return rc.xorRunContainer16(c) + } + panic("unsupported container type") +} + +func (rc *runContainer16) iandNot(a container) container { + switch c := a.(type) { + case *arrayContainer: + return rc.iandNotArray(c) + case *bitmapContainer: + return rc.iandNotBitmap(c) + case *runContainer16: + return rc.iandNotRunContainer16(c) + } + panic("unsupported container type") +} + +// flip the values in the range [firstOfRange,endx) +func (rc *runContainer16) inot(firstOfRange, endx int) container { + if firstOfRange >= endx { + panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange)) + } + // TODO: minimize copies, do it all inplace; not() makes a copy. + rc = rc.Not(firstOfRange, endx) + return rc +} + +func (rc *runContainer16) getCardinality() int { + return int(rc.cardinality()) +} + +func (rc *runContainer16) rank(x uint16) int { + n := int64(len(rc.iv)) + xx := int64(x) + w, already, _ := rc.search(xx, nil) + if w < 0 { + return 0 + } + if !already && w == n-1 { + return rc.getCardinality() + } + var rnk int64 + if !already { + for i := int64(0); i <= w; i++ { + rnk += rc.iv[i].runlen() + } + return int(rnk) + } + for i := int64(0); i < w; i++ { + rnk += rc.iv[i].runlen() + } + rnk += int64(x-rc.iv[w].start) + 1 + return int(rnk) +} + +func (rc *runContainer16) selectInt(x uint16) int { + return rc.selectInt16(x) +} + +func (rc *runContainer16) andNotRunContainer16(b *runContainer16) container { + return rc.AndNotRunContainer16(b) +} + +func (rc *runContainer16) andNotArray(ac *arrayContainer) container { + rcb := rc.toBitmapContainer() + acb := ac.toBitmapContainer() + return rcb.andNotBitmap(acb) +} + +func (rc *runContainer16) andNotBitmap(bc *bitmapContainer) container { + rcb := rc.toBitmapContainer() + return rcb.andNotBitmap(bc) +} + +func (rc *runContainer16) toBitmapContainer() *bitmapContainer { + bc := newBitmapContainer() + for i := range rc.iv { + bc.iaddRange(int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + bc.computeCardinality() + return bc +} + +func (rc *runContainer16) iandNotRunContainer16(x2 *runContainer16) container { + rcb := rc.toBitmapContainer() + x2b := x2.toBitmapContainer() + rcb.iandNotBitmapSurely(x2b) + // TODO: check size and optimize the return value + // TODO: is inplace modification really required? If not, elide the copy. + rc2 := newRunContainer16FromBitmapContainer(rcb) + *rc = *rc2 + return rc +} + +func (rc *runContainer16) iandNotArray(ac *arrayContainer) container { + rcb := rc.toBitmapContainer() + acb := ac.toBitmapContainer() + rcb.iandNotBitmapSurely(acb) + // TODO: check size and optimize the return value + // TODO: is inplace modification really required? If not, elide the copy. + rc2 := newRunContainer16FromBitmapContainer(rcb) + *rc = *rc2 + return rc +} + +func (rc *runContainer16) iandNotBitmap(bc *bitmapContainer) container { + rcb := rc.toBitmapContainer() + rcb.iandNotBitmapSurely(bc) + // TODO: check size and optimize the return value + // TODO: is inplace modification really required? If not, elide the copy. + rc2 := newRunContainer16FromBitmapContainer(rcb) + *rc = *rc2 + return rc +} + +func (rc *runContainer16) xorRunContainer16(x2 *runContainer16) container { + rcb := rc.toBitmapContainer() + x2b := x2.toBitmapContainer() + return rcb.xorBitmap(x2b) +} + +func (rc *runContainer16) xorArray(ac *arrayContainer) container { + rcb := rc.toBitmapContainer() + acb := ac.toBitmapContainer() + return rcb.xorBitmap(acb) +} + +func (rc *runContainer16) xorBitmap(bc *bitmapContainer) container { + rcb := rc.toBitmapContainer() + return rcb.xorBitmap(bc) +} + +// convert to bitmap or array *if needed* +func (rc *runContainer16) toEfficientContainer() container { + + // runContainer16SerializedSizeInBytes(numRuns) + sizeAsRunContainer := rc.getSizeInBytes() + sizeAsBitmapContainer := bitmapContainerSizeInBytes() + card := int(rc.cardinality()) + sizeAsArrayContainer := arrayContainerSizeInBytes(card) + if sizeAsRunContainer <= minOfInt(sizeAsBitmapContainer, sizeAsArrayContainer) { + return rc + } + if card <= arrayDefaultMaxSize { + return rc.toArrayContainer() + } + bc := newBitmapContainerFromRun(rc) + return bc +} + +func (rc *runContainer16) toArrayContainer() *arrayContainer { + ac := newArrayContainer() + for i := range rc.iv { + ac.iaddRange(int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + return ac +} + +func newRunContainer16FromContainer(c container) *runContainer16 { + + switch x := c.(type) { + case *runContainer16: + return x.Clone() + case *arrayContainer: + return newRunContainer16FromArray(x) + case *bitmapContainer: + return newRunContainer16FromBitmapContainer(x) + } + panic("unsupported container type") +} + +// And finds the intersection of rc and b. +func (rc *runContainer16) And(b *Bitmap) *Bitmap { + out := NewBitmap() + for _, p := range rc.iv { + plast := p.last() + for i := p.start; i <= plast; i++ { + if b.Contains(uint32(i)) { + out.Add(uint32(i)) + } + } + } + return out +} + +// Xor returns the exclusive-or of rc and b. +func (rc *runContainer16) Xor(b *Bitmap) *Bitmap { + out := b.Clone() + for _, p := range rc.iv { + plast := p.last() + for v := p.start; v <= plast; v++ { + w := uint32(v) + if out.Contains(w) { + out.RemoveRange(uint64(w), uint64(w+1)) + } else { + out.Add(w) + } + } + } + return out +} + +// Or returns the union of rc and b. +func (rc *runContainer16) Or(b *Bitmap) *Bitmap { + out := b.Clone() + for _, p := range rc.iv { + plast := p.last() + for v := p.start; v <= plast; v++ { + out.Add(uint32(v)) + } + } + return out +} + +// serializedSizeInBytes returns the number of bytes of memory +// required by this runContainer16. This is for the +// Roaring format, as specified https://github.com/RoaringBitmap/RoaringFormatSpec/ +func (rc *runContainer16) serializedSizeInBytes() int { + // number of runs in one uint16, then each run + // needs two more uint16 + return 2 + len(rc.iv)*4 +} + +func (rc *runContainer16) addOffset(x uint16) []container { + low := newRunContainer16() + high := newRunContainer16() + + for _, iv := range rc.iv { + val := int(iv.start) + int(x) + finalVal := int(val) + int(iv.length) + if val <= 0xffff { + if finalVal <= 0xffff { + low.iv = append(low.iv, interval16{uint16(val), iv.length}) + } else { + low.iv = append(low.iv, interval16{uint16(val), uint16(0xffff - val)}) + high.iv = append(high.iv, interval16{uint16(0), uint16(finalVal & 0xffff)}) + } + } else { + high.iv = append(high.iv, interval16{uint16(val & 0xffff), iv.length}) + } + } + return []container{low, high} +} diff --git a/vendor/github.com/RoaringBitmap/roaring/rle16_gen.go b/vendor/github.com/RoaringBitmap/roaring/runcontainer_gen.go index 05bf4463f1..84537d087f 100644 --- a/vendor/github.com/RoaringBitmap/roaring/rle16_gen.go +++ b/vendor/github.com/RoaringBitmap/roaring/runcontainer_gen.go @@ -6,7 +6,7 @@ package roaring import "github.com/tinylib/msgp/msgp" -// DecodeMsg implements msgp.Decodable +// Deprecated: DecodeMsg implements msgp.Decodable func (z *addHelper16) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field @@ -169,7 +169,7 @@ func (z *addHelper16) DecodeMsg(dc *msgp.Reader) (err error) { return } -// EncodeMsg implements msgp.Encodable +// Deprecated: EncodeMsg implements msgp.Encodable func (z *addHelper16) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 5 // write "runstart" @@ -284,7 +284,7 @@ func (z *addHelper16) EncodeMsg(en *msgp.Writer) (err error) { return } -// MarshalMsg implements msgp.Marshaler +// Deprecated: MarshalMsg implements msgp.Marshaler func (z *addHelper16) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 5 @@ -334,7 +334,7 @@ func (z *addHelper16) MarshalMsg(b []byte) (o []byte, err error) { return } -// UnmarshalMsg implements msgp.Unmarshaler +// Deprecated: UnmarshalMsg implements msgp.Unmarshaler func (z *addHelper16) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field @@ -498,7 +498,7 @@ func (z *addHelper16) UnmarshalMsg(bts []byte) (o []byte, err error) { return } -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *addHelper16) Msgsize() (s int) { s = 1 + 9 + msgp.Uint16Size + 7 + msgp.Uint16Size + 14 + msgp.Uint16Size + 2 + msgp.ArrayHeaderSize + (len(z.m) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 3 if z.rc == nil { @@ -509,7 +509,7 @@ func (z *addHelper16) Msgsize() (s int) { return } -// DecodeMsg implements msgp.Decodable +// Deprecated: DecodeMsg implements msgp.Decodable func (z *interval16) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field @@ -546,7 +546,7 @@ func (z *interval16) DecodeMsg(dc *msgp.Reader) (err error) { return } -// EncodeMsg implements msgp.Encodable +// Deprecated: EncodeMsg implements msgp.Encodable func (z interval16) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 2 // write "start" @@ -570,7 +570,7 @@ func (z interval16) EncodeMsg(en *msgp.Writer) (err error) { return } -// MarshalMsg implements msgp.Marshaler +// Deprecated: MarshalMsg implements msgp.Marshaler func (z interval16) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 2 @@ -583,7 +583,7 @@ func (z interval16) MarshalMsg(b []byte) (o []byte, err error) { return } -// UnmarshalMsg implements msgp.Unmarshaler +// Deprecated: UnmarshalMsg implements msgp.Unmarshaler func (z *interval16) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field @@ -621,13 +621,13 @@ func (z *interval16) UnmarshalMsg(bts []byte) (o []byte, err error) { return } -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z interval16) Msgsize() (s int) { s = 1 + 6 + msgp.Uint16Size + 5 + msgp.Uint16Size return } -// DecodeMsg implements msgp.Decodable +// Deprecated: DecodeMsg implements msgp.Decodable func (z *runContainer16) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field @@ -701,7 +701,7 @@ func (z *runContainer16) DecodeMsg(dc *msgp.Reader) (err error) { return } -// EncodeMsg implements msgp.Encodable +// Deprecated: EncodeMsg implements msgp.Encodable func (z *runContainer16) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 2 // write "iv" @@ -746,7 +746,7 @@ func (z *runContainer16) EncodeMsg(en *msgp.Writer) (err error) { return } -// MarshalMsg implements msgp.Marshaler +// Deprecated: MarshalMsg implements msgp.Marshaler func (z *runContainer16) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 2 @@ -768,7 +768,7 @@ func (z *runContainer16) MarshalMsg(b []byte) (o []byte, err error) { return } -// UnmarshalMsg implements msgp.Unmarshaler +// Deprecated: UnmarshalMsg implements msgp.Unmarshaler func (z *runContainer16) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field @@ -843,13 +843,13 @@ func (z *runContainer16) UnmarshalMsg(bts []byte) (o []byte, err error) { return } -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *runContainer16) Msgsize() (s int) { s = 1 + 3 + msgp.ArrayHeaderSize + (len(z.iv) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 5 + msgp.Int64Size return } -// DecodeMsg implements msgp.Decodable +// Deprecated: DecodeMsg implements msgp.Decodable func (z *runIterator16) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field @@ -891,11 +891,6 @@ func (z *runIterator16) DecodeMsg(dc *msgp.Reader) (err error) { if err != nil { return } - case "curSeq": - z.curSeq, err = dc.ReadInt64() - if err != nil { - return - } default: err = dc.Skip() if err != nil { @@ -906,11 +901,11 @@ func (z *runIterator16) DecodeMsg(dc *msgp.Reader) (err error) { return } -// EncodeMsg implements msgp.Encodable +// Deprecated: EncodeMsg implements msgp.Encodable func (z *runIterator16) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 4 + // map header, size 3 // write "rc" - err = en.Append(0x84, 0xa2, 0x72, 0x63) + err = en.Append(0x83, 0xa2, 0x72, 0x63) if err != nil { return err } @@ -943,24 +938,15 @@ func (z *runIterator16) EncodeMsg(en *msgp.Writer) (err error) { if err != nil { return } - // write "curSeq" - err = en.Append(0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71) - if err != nil { - return err - } - err = en.WriteInt64(z.curSeq) - if err != nil { - return - } return } -// MarshalMsg implements msgp.Marshaler +// Deprecated: MarshalMsg implements msgp.Marshaler func (z *runIterator16) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 4 + // map header, size 3 // string "rc" - o = append(o, 0x84, 0xa2, 0x72, 0x63) + o = append(o, 0x83, 0xa2, 0x72, 0x63) if z.rc == nil { o = msgp.AppendNil(o) } else { @@ -975,13 +961,10 @@ func (z *runIterator16) MarshalMsg(b []byte) (o []byte, err error) { // string "curPosInIndex" o = append(o, 0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78) o = msgp.AppendUint16(o, z.curPosInIndex) - // string "curSeq" - o = append(o, 0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71) - o = msgp.AppendInt64(o, z.curSeq) return } -// UnmarshalMsg implements msgp.Unmarshaler +// Deprecated: UnmarshalMsg implements msgp.Unmarshaler func (z *runIterator16) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field @@ -1023,11 +1006,6 @@ func (z *runIterator16) UnmarshalMsg(bts []byte) (o []byte, err error) { if err != nil { return } - case "curSeq": - z.curSeq, bts, err = msgp.ReadInt64Bytes(bts) - if err != nil { - return - } default: bts, err = msgp.Skip(bts) if err != nil { @@ -1039,7 +1017,7 @@ func (z *runIterator16) UnmarshalMsg(bts []byte) (o []byte, err error) { return } -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *runIterator16) Msgsize() (s int) { s = 1 + 3 if z.rc == nil { @@ -1047,11 +1025,11 @@ func (z *runIterator16) Msgsize() (s int) { } else { s += z.rc.Msgsize() } - s += 9 + msgp.Int64Size + 14 + msgp.Uint16Size + 7 + msgp.Int64Size + s += 9 + msgp.Int64Size + 14 + msgp.Uint16Size return } -// DecodeMsg implements msgp.Decodable +// Deprecated: DecodeMsg implements msgp.Decodable func (z *uint16Slice) DecodeMsg(dc *msgp.Reader) (err error) { var zjpj uint32 zjpj, err = dc.ReadArrayHeader() @@ -1072,7 +1050,7 @@ func (z *uint16Slice) DecodeMsg(dc *msgp.Reader) (err error) { return } -// EncodeMsg implements msgp.Encodable +// Deprecated: EncodeMsg implements msgp.Encodable func (z uint16Slice) EncodeMsg(en *msgp.Writer) (err error) { err = en.WriteArrayHeader(uint32(len(z))) if err != nil { @@ -1087,7 +1065,7 @@ func (z uint16Slice) EncodeMsg(en *msgp.Writer) (err error) { return } -// MarshalMsg implements msgp.Marshaler +// Deprecated: MarshalMsg implements msgp.Marshaler func (z uint16Slice) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) o = msgp.AppendArrayHeader(o, uint32(len(z))) @@ -1097,7 +1075,7 @@ func (z uint16Slice) MarshalMsg(b []byte) (o []byte, err error) { return } -// UnmarshalMsg implements msgp.Unmarshaler +// Deprecated: UnmarshalMsg implements msgp.Unmarshaler func (z *uint16Slice) UnmarshalMsg(bts []byte) (o []byte, err error) { var zgmo uint32 zgmo, bts, err = msgp.ReadArrayHeaderBytes(bts) @@ -1119,7 +1097,7 @@ func (z *uint16Slice) UnmarshalMsg(bts []byte) (o []byte, err error) { return } -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z uint16Slice) Msgsize() (s int) { s = msgp.ArrayHeaderSize + (len(z) * (msgp.Uint16Size)) return diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization.go b/vendor/github.com/RoaringBitmap/roaring/serialization.go index 59c39a6630..7b7ed29b0a 100644 --- a/vendor/github.com/RoaringBitmap/roaring/serialization.go +++ b/vendor/github.com/RoaringBitmap/roaring/serialization.go @@ -2,8 +2,6 @@ package roaring import ( "encoding/binary" - "errors" - "fmt" "io" "github.com/tinylib/msgp/msgp" @@ -22,14 +20,6 @@ func (b *runContainer16) writeTo(stream io.Writer) (int, error) { return stream.Write(buf) } -func (b *runContainer32) writeToMsgpack(stream io.Writer) (int, error) { - bts, err := b.MarshalMsg(nil) - if err != nil { - return 0, err - } - return stream.Write(bts) -} - func (b *runContainer16) writeToMsgpack(stream io.Writer) (int, error) { bts, err := b.MarshalMsg(nil) if err != nil { @@ -38,46 +28,7 @@ func (b *runContainer16) writeToMsgpack(stream io.Writer) (int, error) { return stream.Write(bts) } -func (b *runContainer32) readFromMsgpack(stream io.Reader) (int, error) { - err := msgp.Decode(stream, b) - return 0, err -} - func (b *runContainer16) readFromMsgpack(stream io.Reader) (int, error) { err := msgp.Decode(stream, b) return 0, err } - -var errCorruptedStream = errors.New("insufficient/odd number of stored bytes, corrupted stream detected") - -func (b *runContainer16) readFrom(stream io.Reader) (int, error) { - b.iv = b.iv[:0] - b.card = 0 - var numRuns uint16 - err := binary.Read(stream, binary.LittleEndian, &numRuns) - if err != nil { - return 0, err - } - nr := int(numRuns) - encRun := make([]uint16, 2*nr) - by := make([]byte, 4*nr) - err = binary.Read(stream, binary.LittleEndian, &by) - if err != nil { - return 0, err - } - for i := range encRun { - if len(by) < 2 { - return 0, errCorruptedStream - } - encRun[i] = binary.LittleEndian.Uint16(by) - by = by[2:] - } - for i := 0; i < nr; i++ { - if i > 0 && b.iv[i-1].last() >= encRun[i*2] { - return 0, fmt.Errorf("error: stored runContainer had runs that were not in sorted order!! (b.iv[i-1=%v].last = %v >= encRun[i=%v] = %v)", i-1, b.iv[i-1].last(), i, encRun[i*2]) - } - b.iv = append(b.iv, interval16{start: encRun[i*2], length: encRun[i*2+1]}) - b.card += int64(encRun[i*2+1]) + 1 - } - return 0, err -} diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go b/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go index 7fcef7691b..4b9d9e3d48 100644 --- a/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go +++ b/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go @@ -4,6 +4,7 @@ package roaring import ( "encoding/binary" + "errors" "io" ) @@ -26,6 +27,10 @@ func (b *arrayContainer) readFrom(stream io.Reader) (int, error) { } func (b *bitmapContainer) writeTo(stream io.Writer) (int, error) { + if b.cardinality <= arrayDefaultMaxSize { + return 0, errors.New("refusing to write bitmap container with cardinality of array container") + } + // Write set buf := make([]byte, 8*len(b.bitmap)) for i, v := range b.bitmap { @@ -69,6 +74,16 @@ func uint64SliceAsByteSlice(slice []uint64) []byte { return by } +func uint16SliceAsByteSlice(slice []uint16) []byte { + by := make([]byte, len(slice)*2) + + for i, v := range slice { + binary.LittleEndian.PutUint16(by[i*2:], v) + } + + return by +} + func byteSliceAsUint16Slice(slice []byte) []uint16 { if len(slice)%2 != 0 { panic("Slice size should be divisible by 2") diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go index c1d3ad3046..818a06c80b 100644 --- a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go +++ b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go @@ -3,8 +3,10 @@ package roaring import ( + "errors" "io" "reflect" + "runtime" "unsafe" ) @@ -14,26 +16,13 @@ func (ac *arrayContainer) writeTo(stream io.Writer) (int, error) { } func (bc *bitmapContainer) writeTo(stream io.Writer) (int, error) { + if bc.cardinality <= arrayDefaultMaxSize { + return 0, errors.New("refusing to write bitmap container with cardinality of array container") + } buf := uint64SliceAsByteSlice(bc.bitmap) return stream.Write(buf) } -// readFrom reads an arrayContainer from stream. -// PRE-REQUISITE: you must size the arrayContainer correctly (allocate b.content) -// *before* you call readFrom. We can't guess the size in the stream -// by this point. -func (ac *arrayContainer) readFrom(stream io.Reader) (int, error) { - buf := uint16SliceAsByteSlice(ac.content) - return io.ReadFull(stream, buf) -} - -func (bc *bitmapContainer) readFrom(stream io.Reader) (int, error) { - buf := uint64SliceAsByteSlice(bc.bitmap) - n, err := io.ReadFull(stream, buf) - bc.computeCardinality() - return n, err -} - func uint64SliceAsByteSlice(slice []uint64) []byte { // make a new slice header header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) @@ -42,8 +31,12 @@ func uint64SliceAsByteSlice(slice []uint64) []byte { header.Len *= 8 header.Cap *= 8 + // instantiate result and use KeepAlive so data isn't unmapped. + result := *(*[]byte)(unsafe.Pointer(&header)) + runtime.KeepAlive(&slice) + // return it - return *(*[]byte)(unsafe.Pointer(&header)) + return result } func uint16SliceAsByteSlice(slice []uint16) []byte { @@ -54,8 +47,12 @@ func uint16SliceAsByteSlice(slice []uint16) []byte { header.Len *= 2 header.Cap *= 2 + // instantiate result and use KeepAlive so data isn't unmapped. + result := *(*[]byte)(unsafe.Pointer(&header)) + runtime.KeepAlive(&slice) + // return it - return *(*[]byte)(unsafe.Pointer(&header)) + return result } func (bc *bitmapContainer) asLittleEndianByteSlice() []byte { @@ -64,50 +61,74 @@ func (bc *bitmapContainer) asLittleEndianByteSlice() []byte { // Deserialization code follows -func byteSliceAsUint16Slice(slice []byte) []uint16 { +//// +// These methods (byteSliceAsUint16Slice,...) do not make copies, +// they are pointer-based (unsafe). The caller is responsible to +// ensure that the input slice does not get garbage collected, deleted +// or modified while you hold the returned slince. +//// +func byteSliceAsUint16Slice(slice []byte) (result []uint16) { // here we create a new slice holder if len(slice)%2 != 0 { panic("Slice size should be divisible by 2") } + // reference: https://go101.org/article/unsafe.html // make a new slice header - header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) + bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) + rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result)) - // update its capacity and length - header.Len /= 2 - header.Cap /= 2 + // transfer the data from the given slice to a new variable (our result) + rHeader.Data = bHeader.Data + rHeader.Len = bHeader.Len / 2 + rHeader.Cap = bHeader.Cap / 2 - // return it - return *(*[]uint16)(unsafe.Pointer(&header)) + // instantiate result and use KeepAlive so data isn't unmapped. + runtime.KeepAlive(&slice) // it is still crucial, GC can free it) + + // return result + return } -func byteSliceAsUint64Slice(slice []byte) []uint64 { +func byteSliceAsUint64Slice(slice []byte) (result []uint64) { if len(slice)%8 != 0 { panic("Slice size should be divisible by 8") } + // reference: https://go101.org/article/unsafe.html // make a new slice header - header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) + bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) + rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result)) - // update its capacity and length - header.Len /= 8 - header.Cap /= 8 + // transfer the data from the given slice to a new variable (our result) + rHeader.Data = bHeader.Data + rHeader.Len = bHeader.Len / 8 + rHeader.Cap = bHeader.Cap / 8 - // return it - return *(*[]uint64)(unsafe.Pointer(&header)) + // instantiate result and use KeepAlive so data isn't unmapped. + runtime.KeepAlive(&slice) // it is still crucial, GC can free it) + + // return result + return } -func byteSliceAsInterval16Slice(slice []byte) []interval16 { +func byteSliceAsInterval16Slice(slice []byte) (result []interval16) { if len(slice)%4 != 0 { panic("Slice size should be divisible by 4") } + // reference: https://go101.org/article/unsafe.html // make a new slice header - header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) + bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) + rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result)) - // update its capacity and length - header.Len /= 4 - header.Cap /= 4 + // transfer the data from the given slice to a new variable (our result) + rHeader.Data = bHeader.Data + rHeader.Len = bHeader.Len / 4 + rHeader.Cap = bHeader.Cap / 4 - // return it - return *(*[]interval16)(unsafe.Pointer(&header)) + // instantiate result and use KeepAlive so data isn't unmapped. + runtime.KeepAlive(&slice) // it is still crucial, GC can free it) + + // return result + return } diff --git a/vendor/github.com/RoaringBitmap/roaring/shortiterator.go b/vendor/github.com/RoaringBitmap/roaring/shortiterator.go index ef0acbd1ca..15b78bd0c1 100644 --- a/vendor/github.com/RoaringBitmap/roaring/shortiterator.go +++ b/vendor/github.com/RoaringBitmap/roaring/shortiterator.go @@ -5,6 +5,12 @@ type shortIterable interface { next() uint16 } +type shortPeekable interface { + shortIterable + peekNext() uint16 + advanceIfNeeded(minval uint16) +} + type shortIterator struct { slice []uint16 loc int @@ -19,3 +25,28 @@ func (si *shortIterator) next() uint16 { si.loc++ return a } + +func (si *shortIterator) peekNext() uint16 { + return si.slice[si.loc] +} + +func (si *shortIterator) advanceIfNeeded(minval uint16) { + if si.hasNext() && si.peekNext() < minval { + si.loc = advanceUntil(si.slice, si.loc, len(si.slice), minval) + } +} + +type reverseIterator struct { + slice []uint16 + loc int +} + +func (si *reverseIterator) hasNext() bool { + return si.loc >= 0 +} + +func (si *reverseIterator) next() uint16 { + a := si.slice[si.loc] + si.loc-- + return a +} diff --git a/vendor/github.com/RoaringBitmap/roaring/util.go b/vendor/github.com/RoaringBitmap/roaring/util.go index d212660d58..3a9a47236b 100644 --- a/vendor/github.com/RoaringBitmap/roaring/util.go +++ b/vendor/github.com/RoaringBitmap/roaring/util.go @@ -14,6 +14,17 @@ const ( serialCookie = 12347 // runs, arrays, and bitmaps noOffsetThreshold = 4 + // MaxUint32 is the largest uint32 value. + MaxUint32 = 4294967295 + + // MaxRange is One more than the maximum allowed bitmap bit index. For use as an upper + // bound for ranges. + MaxRange uint64 = MaxUint32 + 1 + + // MaxUint16 is the largest 16 bit unsigned int. + // This is the largest value an interval16 can store. + MaxUint16 = 65535 + // Compute wordSizeInBytes, the size of a word in bytes. _m = ^uint64(0) _logS = _m>>8&1 + _m>>16&1 + _m>>32&1 @@ -114,7 +125,6 @@ func flipBitmapRange(bitmap []uint64, start int, end int) { endword := (end - 1) / 64 bitmap[firstword] ^= ^(^uint64(0) << uint(start%64)) for i := firstword; i < endword; i++ { - //p("flipBitmapRange on i=%v", i) bitmap[i] = ^bitmap[i] } bitmap[endword] ^= ^uint64(0) >> (uint(-end) % 64) @@ -292,24 +302,3 @@ func minOfUint16(a, b uint16) uint16 { } return b } - -func maxInt(a, b int) int { - if a > b { - return a - } - return b -} - -func maxUint16(a, b uint16) uint16 { - if a > b { - return a - } - return b -} - -func minUint16(a, b uint16) uint16 { - if a < b { - return a - } - return b -} |