aboutsummaryrefslogtreecommitdiffstats
path: root/vendor/github.com/RoaringBitmap
diff options
context:
space:
mode:
authorMura Li <typeless@users.noreply.github.com>2019-11-27 17:23:33 +0800
committerLauris BH <lauris@nix.lv>2019-11-27 11:23:33 +0200
commit9591185c8f45fefb39e84aa465a943b24ad60a26 (patch)
treec46c697093c98eb5404aa7e2519fbe4ba1a2ad47 /vendor/github.com/RoaringBitmap
parentb50dee5a61b3959bb3606973d359741f0ca9af9a (diff)
downloadgitea-9591185c8f45fefb39e84aa465a943b24ad60a26.tar.gz
gitea-9591185c8f45fefb39e84aa465a943b24ad60a26.zip
Upgrade blevesearch to v0.8.1 (#9177)
For #1441 https://github.com/blevesearch/bleve/commit/a91b427b59b893f112021841ba7370d285f8426f
Diffstat (limited to 'vendor/github.com/RoaringBitmap')
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/.drone.yml20
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/.travis.yml14
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/AUTHORS3
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS6
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/LICENSE33
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/Makefile16
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/README.md9
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/arraycontainer.go34
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go10
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go146
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go20
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/byte_input.go161
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/clz.go11
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/clz_compat.go36
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/go.mod16
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/go.sum30
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/manyiterator.go7
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/parallel.go6
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/rle.go1667
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/rle_gen.go1118
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/rlecommon.go163
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/rlei.go695
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/roaring.go259
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/roaringarray.go246
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go30
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/runcontainer.go (renamed from vendor/github.com/RoaringBitmap/roaring/rle16.go)947
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/runcontainer_gen.go (renamed from vendor/github.com/RoaringBitmap/roaring/rle16_gen.go)82
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/serialization.go49
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/serialization_generic.go15
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go99
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/shortiterator.go31
-rw-r--r--vendor/github.com/RoaringBitmap/roaring/util.go33
32 files changed, 1815 insertions, 4197 deletions
diff --git a/vendor/github.com/RoaringBitmap/roaring/.drone.yml b/vendor/github.com/RoaringBitmap/roaring/.drone.yml
new file mode 100644
index 0000000000..698cd0e7a7
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/.drone.yml
@@ -0,0 +1,20 @@
+kind: pipeline
+name: default
+
+workspace:
+ base: /go
+ path: src/github.com/RoaringBitmap/roaring
+
+steps:
+- name: test
+ image: golang
+ commands:
+ - go get -t
+ - go test
+ - go test -race -run TestConcurrent*
+ - go build -tags appengine
+ - go test -tags appengine
+ - GOARCH=386 go build
+ - GOARCH=386 go test
+ - GOARCH=arm go build
+ - GOARCH=arm64 go build
diff --git a/vendor/github.com/RoaringBitmap/roaring/.travis.yml b/vendor/github.com/RoaringBitmap/roaring/.travis.yml
index 1fdcf3e057..8839c14fd0 100644
--- a/vendor/github.com/RoaringBitmap/roaring/.travis.yml
+++ b/vendor/github.com/RoaringBitmap/roaring/.travis.yml
@@ -8,10 +8,12 @@ install:
notifications:
email: false
go:
-- 1.7.x
-- 1.8.x
-- 1.9.x
-- 1.10.x
+- "1.7.x"
+- "1.8.x"
+- "1.9.x"
+- "1.10.x"
+- "1.11.x"
+- "1.12.x"
- tip
# whitelist
@@ -21,10 +23,14 @@ branches:
script:
- goveralls -v -service travis-ci -ignore arraycontainer_gen.go,bitmapcontainer_gen.go,rle16_gen.go,rle_gen.go,roaringarray_gen.go,rle.go || go test
- go test -race -run TestConcurrent*
+- go build -tags appengine
+- go test -tags appengine
- GOARCH=arm64 go build
- GOARCH=386 go build
- GOARCH=386 go test
- GOARCH=arm go build
+- GOARCH=arm64 go build
+
matrix:
allow_failures:
- go: tip
diff --git a/vendor/github.com/RoaringBitmap/roaring/AUTHORS b/vendor/github.com/RoaringBitmap/roaring/AUTHORS
index 08c074047f..26ec99de9d 100644
--- a/vendor/github.com/RoaringBitmap/roaring/AUTHORS
+++ b/vendor/github.com/RoaringBitmap/roaring/AUTHORS
@@ -7,4 +7,5 @@ Bob Potter (@bpot),
Tyson Maly (@tvmaly),
Will Glynn (@willglynn),
Brent Pedersen (@brentp)
-Maciej Biłas (@maciej)
+Maciej Biłas (@maciej),
+Joe Nall (@joenall)
diff --git a/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS b/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS
index 70b4735dad..b1e3a379f0 100644
--- a/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS
+++ b/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS
@@ -9,4 +9,8 @@ Will Glynn (@willglynn),
Brent Pedersen (@brentp),
Jason E. Aten (@glycerine),
Vali Malinoiu (@0x4139),
-Forud Ghafouri (@fzerorubigd) \ No newline at end of file
+Forud Ghafouri (@fzerorubigd),
+Joe Nall (@joenall),
+(@fredim),
+Edd Robinson (@e-dard),
+Alexander Petrov (@alldroll)
diff --git a/vendor/github.com/RoaringBitmap/roaring/LICENSE b/vendor/github.com/RoaringBitmap/roaring/LICENSE
index aff5f9999b..3ccdd00084 100644
--- a/vendor/github.com/RoaringBitmap/roaring/LICENSE
+++ b/vendor/github.com/RoaringBitmap/roaring/LICENSE
@@ -200,3 +200,36 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+
+================================================================================
+
+Portions of runcontainer.go are from the Go standard library, which is licensed
+under:
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+ * Neither the name of Google Inc. nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/vendor/github.com/RoaringBitmap/roaring/Makefile b/vendor/github.com/RoaringBitmap/roaring/Makefile
index d5259b4c9e..906bd72569 100644
--- a/vendor/github.com/RoaringBitmap/roaring/Makefile
+++ b/vendor/github.com/RoaringBitmap/roaring/Makefile
@@ -1,4 +1,4 @@
-.PHONY: help all test format fmtcheck vet lint qa deps clean nuke rle backrle ser fetch-real-roaring-datasets
+.PHONY: help all test format fmtcheck vet lint qa deps clean nuke ser fetch-real-roaring-datasets
@@ -63,7 +63,7 @@ qa: fmtcheck test vet lint
# Get the dependencies
deps:
- GOPATH=$(GOPATH) go get github.com/smartystreets/goconvey/convey
+ GOPATH=$(GOPATH) go get github.com/stretchr/testify
GOPATH=$(GOPATH) go get github.com/willf/bitset
GOPATH=$(GOPATH) go get github.com/golang/lint/golint
GOPATH=$(GOPATH) go get github.com/mschoch/smat
@@ -97,18 +97,8 @@ nuke:
rm -rf ./target
GOPATH=$(GOPATH) go clean -i ./...
-rle:
- cp rle.go rle16.go
- perl -pi -e 's/32/16/g' rle16.go
- cp rle_test.go rle16_test.go
- perl -pi -e 's/32/16/g' rle16_test.go
-backrle:
- cp rle16.go rle.go
- perl -pi -e 's/16/32/g' rle.go
- perl -pi -e 's/2032/2016/g' rle.go
-
-ser: rle
+ser:
go generate
cover:
diff --git a/vendor/github.com/RoaringBitmap/roaring/README.md b/vendor/github.com/RoaringBitmap/roaring/README.md
index 2c096ce8e6..b711d09ec2 100644
--- a/vendor/github.com/RoaringBitmap/roaring/README.md
+++ b/vendor/github.com/RoaringBitmap/roaring/README.md
@@ -1,4 +1,5 @@
roaring [![Build Status](https://travis-ci.org/RoaringBitmap/roaring.png)](https://travis-ci.org/RoaringBitmap/roaring) [![Coverage Status](https://coveralls.io/repos/github/RoaringBitmap/roaring/badge.svg?branch=master)](https://coveralls.io/github/RoaringBitmap/roaring?branch=master) [![GoDoc](https://godoc.org/github.com/RoaringBitmap/roaring?status.svg)](https://godoc.org/github.com/RoaringBitmap/roaring) [![Go Report Card](https://goreportcard.com/badge/RoaringBitmap/roaring)](https://goreportcard.com/report/github.com/RoaringBitmap/roaring)
+[![Build Status](https://cloud.drone.io/api/badges/RoaringBitmap/roaring/status.svg)](https://cloud.drone.io/RoaringBitmap/roaring)
=============
This is a go version of the Roaring bitmap data structure.
@@ -6,12 +7,12 @@ This is a go version of the Roaring bitmap data structure.
Roaring bitmaps are used by several major systems such as [Apache Lucene][lucene] and derivative systems such as [Solr][solr] and
-[Elasticsearch][elasticsearch], [Metamarkets' Druid][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [Cloud Torrent][cloudtorrent], [Whoosh][whoosh], [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin].
+[Elasticsearch][elasticsearch], [Apache Druid (Incubating)][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [Cloud Torrent][cloudtorrent], [Whoosh][whoosh], [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin].
[lucene]: https://lucene.apache.org/
[solr]: https://lucene.apache.org/solr/
[elasticsearch]: https://www.elastic.co/products/elasticsearch
-[druid]: http://druid.io/
+[druid]: https://druid.apache.org/
[spark]: https://spark.apache.org/
[opensearchserver]: http://www.opensearchserver.com
[cloudtorrent]: https://github.com/jpillora/cloud-torrent
@@ -61,7 +62,6 @@ http://arxiv.org/abs/1402.6407 This paper used data from http://lemire.me/data/r
Dependencies are fetched automatically by giving the `-t` flag to `go get`.
they include
- - github.com/smartystreets/goconvey/convey
- github.com/willf/bitset
- github.com/mschoch/smat
- github.com/glycerine/go-unsnap-stream
@@ -133,6 +133,7 @@ func main() {
if rb1.Equals(newrb) {
fmt.Println("I wrote the content to a byte stream and read it back.")
}
+ // you can iterate over bitmaps using ReverseIterator(), Iterator, ManyIterator()
}
```
@@ -206,7 +207,7 @@ You can use roaring with gore:
- go get -u github.com/motemen/gore
- Make sure that ``$GOPATH/bin`` is in your ``$PATH``.
-- go get github/RoaringBitmap/roaring
+- go get github.com/RoaringBitmap/roaring
```go
$ gore
diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go
index c395868210..621616f5df 100644
--- a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go
+++ b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go
@@ -24,12 +24,16 @@ func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uin
}
}
-func (ac *arrayContainer) getShortIterator() shortIterable {
+func (ac *arrayContainer) getShortIterator() shortPeekable {
return &shortIterator{ac.content, 0}
}
+func (ac *arrayContainer) getReverseIterator() shortIterable {
+ return &reverseIterator{ac.content, len(ac.content) - 1}
+}
+
func (ac *arrayContainer) getManyIterator() manyIterable {
- return &manyIterator{ac.content, 0}
+ return &shortIterator{ac.content, 0}
}
func (ac *arrayContainer) minimum() uint16 {
@@ -115,7 +119,6 @@ func (ac *arrayContainer) iremoveRange(firstOfRange, endx int) container {
// flip the values in the range [firstOfRange,endx)
func (ac *arrayContainer) not(firstOfRange, endx int) container {
if firstOfRange >= endx {
- //p("arrayContainer.not(): exiting early with ac.clone()")
return ac.clone()
}
return ac.notClose(firstOfRange, endx-1) // remove everything in [firstOfRange,endx-1]
@@ -124,18 +127,15 @@ func (ac *arrayContainer) not(firstOfRange, endx int) container {
// flip the values in the range [firstOfRange,lastOfRange]
func (ac *arrayContainer) notClose(firstOfRange, lastOfRange int) container {
if firstOfRange > lastOfRange { // unlike add and remove, not uses an inclusive range [firstOfRange,lastOfRange]
- //p("arrayContainer.notClose(): exiting early with ac.clone()")
return ac.clone()
}
// determine the span of array indices to be affected^M
startIndex := binarySearch(ac.content, uint16(firstOfRange))
- //p("startIndex=%v", startIndex)
if startIndex < 0 {
startIndex = -startIndex - 1
}
lastIndex := binarySearch(ac.content, uint16(lastOfRange))
- //p("lastIndex=%v", lastIndex)
if lastIndex < 0 {
lastIndex = -lastIndex - 2
}
@@ -144,9 +144,7 @@ func (ac *arrayContainer) notClose(firstOfRange, lastOfRange int) container {
newValuesInRange := spanToBeFlipped - currentValuesInRange
cardinalityChange := newValuesInRange - currentValuesInRange
newCardinality := len(ac.content) + cardinalityChange
- //p("new card is %v", newCardinality)
if newCardinality > arrayDefaultMaxSize {
- //p("new card over arrayDefaultMaxSize, so returning bitmap")
return ac.toBitmapContainer().not(firstOfRange, lastOfRange+1)
}
answer := newArrayContainer()
@@ -503,7 +501,6 @@ func (ac *arrayContainer) lazyorArray(value2 *arrayContainer) container {
}
func (ac *arrayContainer) and(a container) container {
- //p("ac.and() called")
switch x := a.(type) {
case *arrayContainer:
return ac.andArray(x)
@@ -550,7 +547,7 @@ func (ac *arrayContainer) iand(a container) container {
return ac.iandBitmap(x)
case *runContainer16:
if x.isFull() {
- return ac.clone()
+ return ac
}
return x.andArray(ac)
}
@@ -722,7 +719,6 @@ func (ac *arrayContainer) inot(firstOfRange, endx int) container {
// flip the values in the range [firstOfRange,lastOfRange]
func (ac *arrayContainer) inotClose(firstOfRange, lastOfRange int) container {
- //p("ac.inotClose() starting")
if firstOfRange > lastOfRange { // unlike add and remove, not uses an inclusive range [firstOfRange,lastOfRange]
return ac
}
@@ -745,7 +741,6 @@ func (ac *arrayContainer) inotClose(firstOfRange, lastOfRange int) container {
if cardinalityChange > 0 {
if newCardinality > len(ac.content) {
if newCardinality > arrayDefaultMaxSize {
- //p("ac.inotClose() converting to bitmap and doing inot there")
bcRet := ac.toBitmapContainer()
bcRet.inot(firstOfRange, lastOfRange+1)
*ac = *bcRet.toArrayContainer()
@@ -766,7 +761,6 @@ func (ac *arrayContainer) inotClose(firstOfRange, lastOfRange int) container {
}
}
ac.content = ac.content[:newCardinality]
- //p("bottom of ac.inotClose(): returning ac")
return ac
}
@@ -958,3 +952,17 @@ func (ac *arrayContainer) toEfficientContainer() container {
func (ac *arrayContainer) containerType() contype {
return arrayContype
}
+
+func (ac *arrayContainer) addOffset(x uint16) []container {
+ low := &arrayContainer{}
+ high := &arrayContainer{}
+ for _, val := range ac.content {
+ y := uint32(val) + uint32(x)
+ if highbits(y) > 0 {
+ high.content = append(high.content, lowbits(y))
+ } else {
+ low.content = append(low.content, lowbits(y))
+ }
+ }
+ return []container{low, high}
+}
diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go
index cba6e53e30..6ee670ee51 100644
--- a/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go
+++ b/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go
@@ -6,7 +6,7 @@ package roaring
import "github.com/tinylib/msgp/msgp"
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
func (z *arrayContainer) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
@@ -49,7 +49,7 @@ func (z *arrayContainer) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
func (z *arrayContainer) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 1
// write "content"
@@ -70,7 +70,7 @@ func (z *arrayContainer) EncodeMsg(en *msgp.Writer) (err error) {
return
}
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
func (z *arrayContainer) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 1
@@ -83,7 +83,7 @@ func (z *arrayContainer) MarshalMsg(b []byte) (o []byte, err error) {
return
}
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
func (z *arrayContainer) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
@@ -127,7 +127,7 @@ func (z *arrayContainer) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *arrayContainer) Msgsize() (s int) {
s = 1 + 8 + msgp.ArrayHeaderSize + (len(z.content) * (msgp.Uint16Size))
return
diff --git a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go
index 5e58b31f2b..e749721bb2 100644
--- a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go
+++ b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go
@@ -110,14 +110,54 @@ func (bcsi *bitmapContainerShortIterator) hasNext() bool {
return bcsi.i >= 0
}
+func (bcsi *bitmapContainerShortIterator) peekNext() uint16 {
+ return uint16(bcsi.i)
+}
+
+func (bcsi *bitmapContainerShortIterator) advanceIfNeeded(minval uint16) {
+ if bcsi.hasNext() && bcsi.peekNext() < minval {
+ bcsi.i = bcsi.ptr.NextSetBit(int(minval))
+ }
+}
+
func newBitmapContainerShortIterator(a *bitmapContainer) *bitmapContainerShortIterator {
return &bitmapContainerShortIterator{a, a.NextSetBit(0)}
}
-func (bc *bitmapContainer) getShortIterator() shortIterable {
+func (bc *bitmapContainer) getShortIterator() shortPeekable {
return newBitmapContainerShortIterator(bc)
}
+type reverseBitmapContainerShortIterator struct {
+ ptr *bitmapContainer
+ i int
+}
+
+func (bcsi *reverseBitmapContainerShortIterator) next() uint16 {
+ if bcsi.i == -1 {
+ panic("reverseBitmapContainerShortIterator.next() going beyond what is available")
+ }
+
+ j := bcsi.i
+ bcsi.i = bcsi.ptr.PrevSetBit(bcsi.i - 1)
+ return uint16(j)
+}
+
+func (bcsi *reverseBitmapContainerShortIterator) hasNext() bool {
+ return bcsi.i >= 0
+}
+
+func newReverseBitmapContainerShortIterator(a *bitmapContainer) *reverseBitmapContainerShortIterator {
+ if a.cardinality == 0 {
+ return &reverseBitmapContainerShortIterator{a, -1}
+ }
+ return &reverseBitmapContainerShortIterator{a, int(a.maximum())}
+}
+
+func (bc *bitmapContainer) getReverseIterator() shortIterable {
+ return newReverseBitmapContainerShortIterator(bc)
+}
+
type bitmapContainerManyIterator struct {
ptr *bitmapContainer
base int
@@ -131,7 +171,7 @@ func (bcmi *bitmapContainerManyIterator) nextMany(hs uint32, buf []uint32) int {
for n < len(buf) {
if bitset == 0 {
- base += 1
+ base++
if base >= len(bcmi.ptr.bitmap) {
bcmi.base = base
bcmi.bitset = bitset
@@ -177,16 +217,13 @@ func bitmapContainerSizeInBytes() int {
func bitmapEquals(a, b []uint64) bool {
if len(a) != len(b) {
- //p("bitmaps differ on length. len(a)=%v; len(b)=%v", len(a), len(b))
return false
}
for i, v := range a {
if v != b[i] {
- //p("bitmaps differ on element i=%v", i)
return false
}
}
- //p("bitmapEquals returning true")
return true
}
@@ -209,9 +246,7 @@ func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask ui
func (bc *bitmapContainer) equals(o container) bool {
srb, ok := o.(*bitmapContainer)
if ok {
- //p("bitmapContainers.equals: both are bitmapContainers")
if srb.cardinality != bc.cardinality {
- //p("bitmapContainers.equals: card differs: %v vs %v", srb.cardinality, bc.cardinality)
return false
}
return bitmapEquals(bc.bitmap, srb.bitmap)
@@ -261,12 +296,6 @@ func (bc *bitmapContainer) iremoveReturnMinimized(i uint16) container {
// iremove returns true if i was found.
func (bc *bitmapContainer) iremove(i uint16) bool {
- /* branchless code
- w := bc.bitmap[i>>6]
- mask := uint64(1) << (i % 64)
- neww := w &^ mask
- bc.cardinality -= int((w ^ neww) >> (i % 64))
- bc.bitmap[i>>6] = neww */
if bc.contains(i) {
bc.cardinality--
bc.bitmap[i/64] &^= (uint64(1) << (i % 64))
@@ -306,14 +335,10 @@ func (bc *bitmapContainer) iremoveRange(firstOfRange, lastOfRange int) container
// flip all values in range [firstOfRange,endx)
func (bc *bitmapContainer) inot(firstOfRange, endx int) container {
- p("bc.inot() called with [%v, %v)", firstOfRange, endx)
if endx-firstOfRange == maxCapacity {
- //p("endx-firstOfRange == maxCapacity")
flipBitmapRange(bc.bitmap, firstOfRange, endx)
bc.cardinality = maxCapacity - bc.cardinality
- //p("bc.cardinality is now %v", bc.cardinality)
} else if endx-firstOfRange > maxCapacity/2 {
- //p("endx-firstOfRange > maxCapacity/2")
flipBitmapRange(bc.bitmap, firstOfRange, endx)
bc.computeCardinality()
} else {
@@ -517,11 +542,31 @@ func (bc *bitmapContainer) iorBitmap(value2 *bitmapContainer) container {
func (bc *bitmapContainer) lazyIORArray(value2 *arrayContainer) container {
answer := bc
c := value2.getCardinality()
- for k := 0; k < c; k++ {
+ for k := 0; k+3 < c; k += 4 {
+ content := (*[4]uint16)(unsafe.Pointer(&value2.content[k]))
+ vc0 := content[0]
+ i0 := uint(vc0) >> 6
+ answer.bitmap[i0] = answer.bitmap[i0] | (uint64(1) << (vc0 % 64))
+
+ vc1 := content[1]
+ i1 := uint(vc1) >> 6
+ answer.bitmap[i1] = answer.bitmap[i1] | (uint64(1) << (vc1 % 64))
+
+ vc2 := content[2]
+ i2 := uint(vc2) >> 6
+ answer.bitmap[i2] = answer.bitmap[i2] | (uint64(1) << (vc2 % 64))
+
+ vc3 := content[3]
+ i3 := uint(vc3) >> 6
+ answer.bitmap[i3] = answer.bitmap[i3] | (uint64(1) << (vc3 % 64))
+ }
+
+ for k := c &^ 3; k < c; k++ {
vc := value2.content[k]
i := uint(vc) >> 6
answer.bitmap[i] = answer.bitmap[i] | (uint64(1) << (vc % 64))
}
+
answer.cardinality = invalidCardinality
return answer
}
@@ -789,8 +834,6 @@ func (bc *bitmapContainer) andNotRun16(rc *runContainer16) container {
}
func (bc *bitmapContainer) iandNot(a container) container {
- //p("bitmapContainer.iandNot() starting")
-
switch x := a.(type) {
case *arrayContainer:
return bc.iandNotArray(x)
@@ -844,12 +887,15 @@ func (bc *bitmapContainer) andNotBitmap(value2 *bitmapContainer) container {
return ac
}
-func (bc *bitmapContainer) iandNotBitmapSurely(value2 *bitmapContainer) *bitmapContainer {
+func (bc *bitmapContainer) iandNotBitmapSurely(value2 *bitmapContainer) container {
newCardinality := int(popcntMaskSlice(bc.bitmap, value2.bitmap))
for k := 0; k < len(bc.bitmap); k++ {
bc.bitmap[k] = bc.bitmap[k] &^ value2.bitmap[k]
}
bc.cardinality = newCardinality
+ if bc.getCardinality() <= arrayDefaultMaxSize {
+ return bc.toArrayContainer()
+ }
return bc
}
@@ -917,6 +963,32 @@ func (bc *bitmapContainer) NextSetBit(i int) int {
return -1
}
+func (bc *bitmapContainer) PrevSetBit(i int) int {
+ if i < 0 {
+ return -1
+ }
+ x := i / 64
+ if x >= len(bc.bitmap) {
+ return -1
+ }
+
+ w := bc.bitmap[x]
+
+ b := i % 64
+
+ w = w << uint(63-b)
+ if w != 0 {
+ return i - countLeadingZeros(w)
+ }
+ x--
+ for ; x >= 0; x-- {
+ if bc.bitmap[x] != 0 {
+ return (x * 64) + 63 - countLeadingZeros(bc.bitmap[x])
+ }
+ }
+ return -1
+}
+
// reference the java implementation
// https://github.com/RoaringBitmap/RoaringBitmap/blob/master/src/main/java/org/roaringbitmap/BitmapContainer.java#L875-L892
//
@@ -980,3 +1052,35 @@ func newBitmapContainerFromRun(rc *runContainer16) *bitmapContainer {
func (bc *bitmapContainer) containerType() contype {
return bitmapContype
}
+
+func (bc *bitmapContainer) addOffset(x uint16) []container {
+ low := newBitmapContainer()
+ high := newBitmapContainer()
+ b := uint32(x) >> 6
+ i := uint32(x) % 64
+ end := uint32(1024) - b
+ if i == 0 {
+ copy(low.bitmap[b:], bc.bitmap[:end])
+ copy(high.bitmap[:b], bc.bitmap[end:])
+ } else {
+ low.bitmap[b] = bc.bitmap[0] << i
+ for k := uint32(1); k < end; k++ {
+ newval := bc.bitmap[k] << i
+ if newval == 0 {
+ newval = bc.bitmap[k-1] >> (64 - i)
+ }
+ low.bitmap[b+k] = newval
+ }
+ for k := end; k < 1024; k++ {
+ newval := bc.bitmap[k] << i
+ if newval == 0 {
+ newval = bc.bitmap[k-1] >> (64 - i)
+ }
+ high.bitmap[k-end] = newval
+ }
+ high.bitmap[b] = bc.bitmap[1023] >> (64 - i)
+ }
+ low.computeCardinality()
+ high.computeCardinality()
+ return []container{low, high}
+}
diff --git a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go
index f6c053e650..9b5a465f38 100644
--- a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go
+++ b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go
@@ -6,7 +6,7 @@ package roaring
import "github.com/tinylib/msgp/msgp"
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
func (z *bitmapContainer) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
@@ -54,7 +54,7 @@ func (z *bitmapContainer) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
func (z *bitmapContainer) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 2
// write "cardinality"
@@ -84,7 +84,7 @@ func (z *bitmapContainer) EncodeMsg(en *msgp.Writer) (err error) {
return
}
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
func (z *bitmapContainer) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 2
@@ -100,7 +100,7 @@ func (z *bitmapContainer) MarshalMsg(b []byte) (o []byte, err error) {
return
}
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
func (z *bitmapContainer) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
@@ -149,13 +149,13 @@ func (z *bitmapContainer) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *bitmapContainer) Msgsize() (s int) {
s = 1 + 12 + msgp.IntSize + 7 + msgp.ArrayHeaderSize + (len(z.bitmap) * (msgp.Uint64Size))
return
}
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
func (z *bitmapContainerShortIterator) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
@@ -239,7 +239,7 @@ func (z *bitmapContainerShortIterator) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
func (z *bitmapContainerShortIterator) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 2
// write "ptr"
@@ -291,7 +291,7 @@ func (z *bitmapContainerShortIterator) EncodeMsg(en *msgp.Writer) (err error) {
return
}
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
func (z *bitmapContainerShortIterator) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 2
@@ -317,7 +317,7 @@ func (z *bitmapContainerShortIterator) MarshalMsg(b []byte) (o []byte, err error
return
}
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
func (z *bitmapContainerShortIterator) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
@@ -402,7 +402,7 @@ func (z *bitmapContainerShortIterator) UnmarshalMsg(bts []byte) (o []byte, err e
return
}
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *bitmapContainerShortIterator) Msgsize() (s int) {
s = 1 + 4
if z.ptr == nil {
diff --git a/vendor/github.com/RoaringBitmap/roaring/byte_input.go b/vendor/github.com/RoaringBitmap/roaring/byte_input.go
new file mode 100644
index 0000000000..f7a98a1d40
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/byte_input.go
@@ -0,0 +1,161 @@
+package roaring
+
+import (
+ "encoding/binary"
+ "io"
+)
+
+type byteInput interface {
+ // next returns a slice containing the next n bytes from the buffer,
+ // advancing the buffer as if the bytes had been returned by Read.
+ next(n int) ([]byte, error)
+ // readUInt32 reads uint32 with LittleEndian order
+ readUInt32() (uint32, error)
+ // readUInt16 reads uint16 with LittleEndian order
+ readUInt16() (uint16, error)
+ // getReadBytes returns read bytes
+ getReadBytes() int64
+ // skipBytes skips exactly n bytes
+ skipBytes(n int) error
+}
+
+func newByteInputFromReader(reader io.Reader) byteInput {
+ return &byteInputAdapter{
+ r: reader,
+ readBytes: 0,
+ }
+}
+
+func newByteInput(buf []byte) byteInput {
+ return &byteBuffer{
+ buf: buf,
+ off: 0,
+ }
+}
+
+type byteBuffer struct {
+ buf []byte
+ off int
+}
+
+// next returns a slice containing the next n bytes from the reader
+// If there are fewer bytes than the given n, io.ErrUnexpectedEOF will be returned
+func (b *byteBuffer) next(n int) ([]byte, error) {
+ m := len(b.buf) - b.off
+
+ if n > m {
+ return nil, io.ErrUnexpectedEOF
+ }
+
+ data := b.buf[b.off : b.off+n]
+ b.off += n
+
+ return data, nil
+}
+
+// readUInt32 reads uint32 with LittleEndian order
+func (b *byteBuffer) readUInt32() (uint32, error) {
+ if len(b.buf)-b.off < 4 {
+ return 0, io.ErrUnexpectedEOF
+ }
+
+ v := binary.LittleEndian.Uint32(b.buf[b.off:])
+ b.off += 4
+
+ return v, nil
+}
+
+// readUInt16 reads uint16 with LittleEndian order
+func (b *byteBuffer) readUInt16() (uint16, error) {
+ if len(b.buf)-b.off < 2 {
+ return 0, io.ErrUnexpectedEOF
+ }
+
+ v := binary.LittleEndian.Uint16(b.buf[b.off:])
+ b.off += 2
+
+ return v, nil
+}
+
+// getReadBytes returns read bytes
+func (b *byteBuffer) getReadBytes() int64 {
+ return int64(b.off)
+}
+
+// skipBytes skips exactly n bytes
+func (b *byteBuffer) skipBytes(n int) error {
+ m := len(b.buf) - b.off
+
+ if n > m {
+ return io.ErrUnexpectedEOF
+ }
+
+ b.off += n
+
+ return nil
+}
+
+// reset resets the given buffer with a new byte slice
+func (b *byteBuffer) reset(buf []byte) {
+ b.buf = buf
+ b.off = 0
+}
+
+type byteInputAdapter struct {
+ r io.Reader
+ readBytes int
+}
+
+// next returns a slice containing the next n bytes from the buffer,
+// advancing the buffer as if the bytes had been returned by Read.
+func (b *byteInputAdapter) next(n int) ([]byte, error) {
+ buf := make([]byte, n)
+ m, err := io.ReadAtLeast(b.r, buf, n)
+ b.readBytes += m
+
+ if err != nil {
+ return nil, err
+ }
+
+ return buf, nil
+}
+
+// readUInt32 reads uint32 with LittleEndian order
+func (b *byteInputAdapter) readUInt32() (uint32, error) {
+ buf, err := b.next(4)
+
+ if err != nil {
+ return 0, err
+ }
+
+ return binary.LittleEndian.Uint32(buf), nil
+}
+
+// readUInt16 reads uint16 with LittleEndian order
+func (b *byteInputAdapter) readUInt16() (uint16, error) {
+ buf, err := b.next(2)
+
+ if err != nil {
+ return 0, err
+ }
+
+ return binary.LittleEndian.Uint16(buf), nil
+}
+
+// getReadBytes returns read bytes
+func (b *byteInputAdapter) getReadBytes() int64 {
+ return int64(b.readBytes)
+}
+
+// skipBytes skips exactly n bytes
+func (b *byteInputAdapter) skipBytes(n int) error {
+ _, err := b.next(n)
+
+ return err
+}
+
+// reset resets the given buffer with a new stream
+func (b *byteInputAdapter) reset(stream io.Reader) {
+ b.r = stream
+ b.readBytes = 0
+}
diff --git a/vendor/github.com/RoaringBitmap/roaring/clz.go b/vendor/github.com/RoaringBitmap/roaring/clz.go
new file mode 100644
index 0000000000..bcd80d32f0
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/clz.go
@@ -0,0 +1,11 @@
+// +build go1.9
+// "go1.9", from Go version 1.9 onward
+// See https://golang.org/pkg/go/build/#hdr-Build_Constraints
+
+package roaring
+
+import "math/bits"
+
+func countLeadingZeros(x uint64) int {
+ return bits.LeadingZeros64(x)
+}
diff --git a/vendor/github.com/RoaringBitmap/roaring/clz_compat.go b/vendor/github.com/RoaringBitmap/roaring/clz_compat.go
new file mode 100644
index 0000000000..eeef4de35b
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/clz_compat.go
@@ -0,0 +1,36 @@
+// +build !go1.9
+
+package roaring
+
+// LeadingZeroBits returns the number of consecutive most significant zero
+// bits of x.
+func countLeadingZeros(i uint64) int {
+ if i == 0 {
+ return 64
+ }
+ n := 1
+ x := uint32(i >> 32)
+ if x == 0 {
+ n += 32
+ x = uint32(i)
+ }
+ if (x >> 16) == 0 {
+ n += 16
+ x <<= 16
+ }
+ if (x >> 24) == 0 {
+ n += 8
+ x <<= 8
+ }
+ if x>>28 == 0 {
+ n += 4
+ x <<= 4
+ }
+ if x>>30 == 0 {
+ n += 2
+ x <<= 2
+
+ }
+ n -= int(x >> 31)
+ return n
+}
diff --git a/vendor/github.com/RoaringBitmap/roaring/go.mod b/vendor/github.com/RoaringBitmap/roaring/go.mod
new file mode 100644
index 0000000000..f5aebf3967
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/go.mod
@@ -0,0 +1,16 @@
+module github.com/RoaringBitmap/roaring
+
+go 1.12
+
+require (
+ github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2
+ github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect
+ github.com/golang/snappy v0.0.1 // indirect
+ github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 // indirect
+ github.com/jtolds/gls v4.20.0+incompatible // indirect
+ github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae
+ github.com/philhofer/fwd v1.0.0 // indirect
+ github.com/stretchr/testify v1.4.0
+ github.com/tinylib/msgp v1.1.0
+ github.com/willf/bitset v1.1.10
+)
diff --git a/vendor/github.com/RoaringBitmap/roaring/go.sum b/vendor/github.com/RoaringBitmap/roaring/go.sum
new file mode 100644
index 0000000000..2e27dbb6e6
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/go.sum
@@ -0,0 +1,30 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4=
+github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
+github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8=
+github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 h1:twflg0XRTjwKpxb/jFExr4HGq6on2dEOmnL6FV+fgPw=
+github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY=
+github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
+github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
+github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
+github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
+github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/vendor/github.com/RoaringBitmap/roaring/manyiterator.go b/vendor/github.com/RoaringBitmap/roaring/manyiterator.go
index b4f630a7b4..3007563775 100644
--- a/vendor/github.com/RoaringBitmap/roaring/manyiterator.go
+++ b/vendor/github.com/RoaringBitmap/roaring/manyiterator.go
@@ -4,12 +4,7 @@ type manyIterable interface {
nextMany(hs uint32, buf []uint32) int
}
-type manyIterator struct {
- slice []uint16
- loc int
-}
-
-func (si *manyIterator) nextMany(hs uint32, buf []uint32) int {
+func (si *shortIterator) nextMany(hs uint32, buf []uint32) int {
n := 0
l := si.loc
s := si.slice
diff --git a/vendor/github.com/RoaringBitmap/roaring/parallel.go b/vendor/github.com/RoaringBitmap/roaring/parallel.go
index 09f94fe83c..2af1aed48e 100644
--- a/vendor/github.com/RoaringBitmap/roaring/parallel.go
+++ b/vendor/github.com/RoaringBitmap/roaring/parallel.go
@@ -143,8 +143,8 @@ func toBitmapContainer(c container) container {
func appenderRoutine(bitmapChan chan<- *Bitmap, resultChan <-chan keyedContainer, expectedKeysChan <-chan int) {
expectedKeys := -1
appendedKeys := 0
- keys := make([]uint16, 0)
- containers := make([]container, 0)
+ var keys []uint16
+ var containers []container
for appendedKeys != expectedKeys {
select {
case item := <-resultChan:
@@ -337,7 +337,7 @@ func ParAnd(parallelism int, bitmaps ...*Bitmap) *Bitmap {
// (if it is set to 0, a default number of workers is chosen)
func ParOr(parallelism int, bitmaps ...*Bitmap) *Bitmap {
var lKey uint16 = MaxUint16
- var hKey uint16 = 0
+ var hKey uint16
bitmapsFiltered := bitmaps[:0]
for _, b := range bitmaps {
diff --git a/vendor/github.com/RoaringBitmap/roaring/rle.go b/vendor/github.com/RoaringBitmap/roaring/rle.go
deleted file mode 100644
index 8f3d4edd68..0000000000
--- a/vendor/github.com/RoaringBitmap/roaring/rle.go
+++ /dev/null
@@ -1,1667 +0,0 @@
-package roaring
-
-//
-// Copyright (c) 2016 by the roaring authors.
-// Licensed under the Apache License, Version 2.0.
-//
-// We derive a few lines of code from the sort.Search
-// function in the golang standard library. That function
-// is Copyright 2009 The Go Authors, and licensed
-// under the following BSD-style license.
-/*
-Copyright (c) 2009 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-import (
- "fmt"
- "sort"
- "unsafe"
-)
-
-//go:generate msgp -unexported
-
-// runContainer32 does run-length encoding of sets of
-// uint32 integers.
-type runContainer32 struct {
- iv []interval32
- card int64
-
- // avoid allocation during search
- myOpts searchOptions `msg:"-"`
-}
-
-// interval32 is the internal to runContainer32
-// structure that maintains the individual [Start, last]
-// closed intervals.
-type interval32 struct {
- start uint32
- last uint32
-}
-
-// runlen returns the count of integers in the interval.
-func (iv interval32) runlen() int64 {
- return 1 + int64(iv.last) - int64(iv.start)
-}
-
-// String produces a human viewable string of the contents.
-func (iv interval32) String() string {
- return fmt.Sprintf("[%d, %d]", iv.start, iv.last)
-}
-
-func ivalString32(iv []interval32) string {
- var s string
- var j int
- var p interval32
- for j, p = range iv {
- s += fmt.Sprintf("%v:[%d, %d], ", j, p.start, p.last)
- }
- return s
-}
-
-// String produces a human viewable string of the contents.
-func (rc *runContainer32) String() string {
- if len(rc.iv) == 0 {
- return "runContainer32{}"
- }
- is := ivalString32(rc.iv)
- return `runContainer32{` + is + `}`
-}
-
-// uint32Slice is a sort.Sort convenience method
-type uint32Slice []uint32
-
-// Len returns the length of p.
-func (p uint32Slice) Len() int { return len(p) }
-
-// Less returns p[i] < p[j]
-func (p uint32Slice) Less(i, j int) bool { return p[i] < p[j] }
-
-// Swap swaps elements i and j.
-func (p uint32Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
-
-//msgp:ignore addHelper
-
-// addHelper helps build a runContainer32.
-type addHelper32 struct {
- runstart uint32
- runlen uint32
- actuallyAdded uint32
- m []interval32
- rc *runContainer32
-}
-
-func (ah *addHelper32) storeIval(runstart, runlen uint32) {
- mi := interval32{start: runstart, last: runstart + runlen}
- ah.m = append(ah.m, mi)
-}
-
-func (ah *addHelper32) add(cur, prev uint32, i int) {
- if cur == prev+1 {
- ah.runlen++
- ah.actuallyAdded++
- } else {
- if cur < prev {
- panic(fmt.Sprintf("newRunContainer32FromVals sees "+
- "unsorted vals; vals[%v]=cur=%v < prev=%v. Sort your vals"+
- " before calling us with alreadySorted == true.", i, cur, prev))
- }
- if cur == prev {
- // ignore duplicates
- } else {
- ah.actuallyAdded++
- ah.storeIval(ah.runstart, ah.runlen)
- ah.runstart = cur
- ah.runlen = 0
- }
- }
-}
-
-// newRunContainerRange makes a new container made of just the specified closed interval [rangestart,rangelast]
-func newRunContainer32Range(rangestart uint32, rangelast uint32) *runContainer32 {
- rc := &runContainer32{}
- rc.iv = append(rc.iv, interval32{start: rangestart, last: rangelast})
- return rc
-}
-
-// newRunContainer32FromVals makes a new container from vals.
-//
-// For efficiency, vals should be sorted in ascending order.
-// Ideally vals should not contain duplicates, but we detect and
-// ignore them. If vals is already sorted in ascending order, then
-// pass alreadySorted = true. Otherwise, for !alreadySorted,
-// we will sort vals before creating a runContainer32 of them.
-// We sort the original vals, so this will change what the
-// caller sees in vals as a side effect.
-func newRunContainer32FromVals(alreadySorted bool, vals ...uint32) *runContainer32 {
- // keep this in sync with newRunContainer32FromArray below
-
- rc := &runContainer32{}
- ah := addHelper32{rc: rc}
-
- if !alreadySorted {
- sort.Sort(uint32Slice(vals))
- }
- n := len(vals)
- var cur, prev uint32
- switch {
- case n == 0:
- // nothing more
- case n == 1:
- ah.m = append(ah.m, interval32{start: vals[0], last: vals[0]})
- ah.actuallyAdded++
- default:
- ah.runstart = vals[0]
- ah.actuallyAdded++
- for i := 1; i < n; i++ {
- prev = vals[i-1]
- cur = vals[i]
- ah.add(cur, prev, i)
- }
- ah.storeIval(ah.runstart, ah.runlen)
- }
- rc.iv = ah.m
- rc.card = int64(ah.actuallyAdded)
- return rc
-}
-
-// newRunContainer32FromBitmapContainer makes a new run container from bc,
-// somewhat efficiently. For reference, see the Java
-// https://github.com/RoaringBitmap/RoaringBitmap/blob/master/src/main/java/org/roaringbitmap/RunContainer.java#L145-L192
-func newRunContainer32FromBitmapContainer(bc *bitmapContainer) *runContainer32 {
-
- rc := &runContainer32{}
- nbrRuns := bc.numberOfRuns()
- if nbrRuns == 0 {
- return rc
- }
- rc.iv = make([]interval32, nbrRuns)
-
- longCtr := 0 // index of current long in bitmap
- curWord := bc.bitmap[0] // its value
- runCount := 0
- for {
- // potentially multiword advance to first 1 bit
- for curWord == 0 && longCtr < len(bc.bitmap)-1 {
- longCtr++
- curWord = bc.bitmap[longCtr]
- }
-
- if curWord == 0 {
- // wrap up, no more runs
- return rc
- }
- localRunStart := countTrailingZeros(curWord)
- runStart := localRunStart + 64*longCtr
- // stuff 1s into number's LSBs
- curWordWith1s := curWord | (curWord - 1)
-
- // find the next 0, potentially in a later word
- runEnd := 0
- for curWordWith1s == maxWord && longCtr < len(bc.bitmap)-1 {
- longCtr++
- curWordWith1s = bc.bitmap[longCtr]
- }
-
- if curWordWith1s == maxWord {
- // a final unterminated run of 1s
- runEnd = wordSizeInBits + longCtr*64
- rc.iv[runCount].start = uint32(runStart)
- rc.iv[runCount].last = uint32(runEnd) - 1
- return rc
- }
- localRunEnd := countTrailingZeros(^curWordWith1s)
- runEnd = localRunEnd + longCtr*64
- rc.iv[runCount].start = uint32(runStart)
- rc.iv[runCount].last = uint32(runEnd) - 1
- runCount++
- // now, zero out everything right of runEnd.
- curWord = curWordWith1s & (curWordWith1s + 1)
- // We've lathered and rinsed, so repeat...
- }
-
-}
-
-//
-// newRunContainer32FromArray populates a new
-// runContainer32 from the contents of arr.
-//
-func newRunContainer32FromArray(arr *arrayContainer) *runContainer32 {
- // keep this in sync with newRunContainer32FromVals above
-
- rc := &runContainer32{}
- ah := addHelper32{rc: rc}
-
- n := arr.getCardinality()
- var cur, prev uint32
- switch {
- case n == 0:
- // nothing more
- case n == 1:
- ah.m = append(ah.m, interval32{start: uint32(arr.content[0]), last: uint32(arr.content[0])})
- ah.actuallyAdded++
- default:
- ah.runstart = uint32(arr.content[0])
- ah.actuallyAdded++
- for i := 1; i < n; i++ {
- prev = uint32(arr.content[i-1])
- cur = uint32(arr.content[i])
- ah.add(cur, prev, i)
- }
- ah.storeIval(ah.runstart, ah.runlen)
- }
- rc.iv = ah.m
- rc.card = int64(ah.actuallyAdded)
- return rc
-}
-
-// set adds the integers in vals to the set. Vals
-// must be sorted in increasing order; if not, you should set
-// alreadySorted to false, and we will sort them in place for you.
-// (Be aware of this side effect -- it will affect the callers
-// view of vals).
-//
-// If you have a small number of additions to an already
-// big runContainer32, calling Add() may be faster.
-func (rc *runContainer32) set(alreadySorted bool, vals ...uint32) {
-
- rc2 := newRunContainer32FromVals(alreadySorted, vals...)
- un := rc.union(rc2)
- rc.iv = un.iv
- rc.card = 0
-}
-
-// canMerge returns true if the intervals
-// a and b either overlap or they are
-// contiguous and so can be merged into
-// a single interval.
-func canMerge32(a, b interval32) bool {
- if int64(a.last)+1 < int64(b.start) {
- return false
- }
- return int64(b.last)+1 >= int64(a.start)
-}
-
-// haveOverlap differs from canMerge in that
-// it tells you if the intersection of a
-// and b would contain an element (otherwise
-// it would be the empty set, and we return
-// false).
-func haveOverlap32(a, b interval32) bool {
- if int64(a.last)+1 <= int64(b.start) {
- return false
- }
- return int64(b.last)+1 > int64(a.start)
-}
-
-// mergeInterval32s joins a and b into a
-// new interval, and panics if it cannot.
-func mergeInterval32s(a, b interval32) (res interval32) {
- if !canMerge32(a, b) {
- panic(fmt.Sprintf("cannot merge %#v and %#v", a, b))
- }
- if b.start < a.start {
- res.start = b.start
- } else {
- res.start = a.start
- }
- if b.last > a.last {
- res.last = b.last
- } else {
- res.last = a.last
- }
- return
-}
-
-// intersectInterval32s returns the intersection
-// of a and b. The isEmpty flag will be true if
-// a and b were disjoint.
-func intersectInterval32s(a, b interval32) (res interval32, isEmpty bool) {
- if !haveOverlap32(a, b) {
- isEmpty = true
- return
- }
- if b.start > a.start {
- res.start = b.start
- } else {
- res.start = a.start
- }
- if b.last < a.last {
- res.last = b.last
- } else {
- res.last = a.last
- }
- return
-}
-
-// union merges two runContainer32s, producing
-// a new runContainer32 with the union of rc and b.
-func (rc *runContainer32) union(b *runContainer32) *runContainer32 {
-
- // rc is also known as 'a' here, but golint insisted we
- // call it rc for consistency with the rest of the methods.
-
- var m []interval32
-
- alim := int64(len(rc.iv))
- blim := int64(len(b.iv))
-
- var na int64 // next from a
- var nb int64 // next from b
-
- // merged holds the current merge output, which might
- // get additional merges before being appended to m.
- var merged interval32
- var mergedUsed bool // is merged being used at the moment?
-
- var cura interval32 // currently considering this interval32 from a
- var curb interval32 // currently considering this interval32 from b
-
- pass := 0
- for na < alim && nb < blim {
- pass++
- cura = rc.iv[na]
- curb = b.iv[nb]
-
- if mergedUsed {
- mergedUpdated := false
- if canMerge32(cura, merged) {
- merged = mergeInterval32s(cura, merged)
- na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1)
- mergedUpdated = true
- }
- if canMerge32(curb, merged) {
- merged = mergeInterval32s(curb, merged)
- nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1)
- mergedUpdated = true
- }
- if !mergedUpdated {
- // we know that merged is disjoint from cura and curb
- m = append(m, merged)
- mergedUsed = false
- }
- continue
-
- } else {
- // !mergedUsed
- if !canMerge32(cura, curb) {
- if cura.start < curb.start {
- m = append(m, cura)
- na++
- } else {
- m = append(m, curb)
- nb++
- }
- } else {
- merged = mergeInterval32s(cura, curb)
- mergedUsed = true
- na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1)
- nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1)
- }
- }
- }
- var aDone, bDone bool
- if na >= alim {
- aDone = true
- }
- if nb >= blim {
- bDone = true
- }
- // finish by merging anything remaining into merged we can:
- if mergedUsed {
- if !aDone {
- aAdds:
- for na < alim {
- cura = rc.iv[na]
- if canMerge32(cura, merged) {
- merged = mergeInterval32s(cura, merged)
- na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1)
- } else {
- break aAdds
- }
- }
-
- }
-
- if !bDone {
- bAdds:
- for nb < blim {
- curb = b.iv[nb]
- if canMerge32(curb, merged) {
- merged = mergeInterval32s(curb, merged)
- nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1)
- } else {
- break bAdds
- }
- }
-
- }
-
- m = append(m, merged)
- }
- if na < alim {
- m = append(m, rc.iv[na:]...)
- }
- if nb < blim {
- m = append(m, b.iv[nb:]...)
- }
-
- res := &runContainer32{iv: m}
- return res
-}
-
-// unionCardinality returns the cardinality of the merger of two runContainer32s, the union of rc and b.
-func (rc *runContainer32) unionCardinality(b *runContainer32) uint64 {
-
- // rc is also known as 'a' here, but golint insisted we
- // call it rc for consistency with the rest of the methods.
- answer := uint64(0)
-
- alim := int64(len(rc.iv))
- blim := int64(len(b.iv))
-
- var na int64 // next from a
- var nb int64 // next from b
-
- // merged holds the current merge output, which might
- // get additional merges before being appended to m.
- var merged interval32
- var mergedUsed bool // is merged being used at the moment?
-
- var cura interval32 // currently considering this interval32 from a
- var curb interval32 // currently considering this interval32 from b
-
- pass := 0
- for na < alim && nb < blim {
- pass++
- cura = rc.iv[na]
- curb = b.iv[nb]
-
- if mergedUsed {
- mergedUpdated := false
- if canMerge32(cura, merged) {
- merged = mergeInterval32s(cura, merged)
- na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1)
- mergedUpdated = true
- }
- if canMerge32(curb, merged) {
- merged = mergeInterval32s(curb, merged)
- nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1)
- mergedUpdated = true
- }
- if !mergedUpdated {
- // we know that merged is disjoint from cura and curb
- //m = append(m, merged)
- answer += uint64(merged.last) - uint64(merged.start) + 1
- mergedUsed = false
- }
- continue
-
- } else {
- // !mergedUsed
- if !canMerge32(cura, curb) {
- if cura.start < curb.start {
- answer += uint64(cura.last) - uint64(cura.start) + 1
- //m = append(m, cura)
- na++
- } else {
- answer += uint64(curb.last) - uint64(curb.start) + 1
- //m = append(m, curb)
- nb++
- }
- } else {
- merged = mergeInterval32s(cura, curb)
- mergedUsed = true
- na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1)
- nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1)
- }
- }
- }
- var aDone, bDone bool
- if na >= alim {
- aDone = true
- }
- if nb >= blim {
- bDone = true
- }
- // finish by merging anything remaining into merged we can:
- if mergedUsed {
- if !aDone {
- aAdds:
- for na < alim {
- cura = rc.iv[na]
- if canMerge32(cura, merged) {
- merged = mergeInterval32s(cura, merged)
- na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1)
- } else {
- break aAdds
- }
- }
-
- }
-
- if !bDone {
- bAdds:
- for nb < blim {
- curb = b.iv[nb]
- if canMerge32(curb, merged) {
- merged = mergeInterval32s(curb, merged)
- nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1)
- } else {
- break bAdds
- }
- }
-
- }
-
- //m = append(m, merged)
- answer += uint64(merged.last) - uint64(merged.start) + 1
- }
- for _, r := range rc.iv[na:] {
- answer += uint64(r.last) - uint64(r.start) + 1
- }
- for _, r := range b.iv[nb:] {
- answer += uint64(r.last) - uint64(r.start) + 1
- }
- return answer
-}
-
-// indexOfIntervalAtOrAfter is a helper for union.
-func (rc *runContainer32) indexOfIntervalAtOrAfter(key int64, startIndex int64) int64 {
- rc.myOpts.startIndex = startIndex
- rc.myOpts.endxIndex = 0
-
- w, already, _ := rc.search(key, &rc.myOpts)
- if already {
- return w
- }
- return w + 1
-}
-
-// intersect returns a new runContainer32 holding the
-// intersection of rc (also known as 'a') and b.
-func (rc *runContainer32) intersect(b *runContainer32) *runContainer32 {
-
- a := rc
- numa := int64(len(a.iv))
- numb := int64(len(b.iv))
- res := &runContainer32{}
- if numa == 0 || numb == 0 {
- return res
- }
-
- if numa == 1 && numb == 1 {
- if !haveOverlap32(a.iv[0], b.iv[0]) {
- return res
- }
- }
-
- var output []interval32
-
- var acuri int64
- var bcuri int64
-
- astart := int64(a.iv[acuri].start)
- bstart := int64(b.iv[bcuri].start)
-
- var intersection interval32
- var leftoverstart int64
- var isOverlap, isLeftoverA, isLeftoverB bool
- var done bool
- pass := 0
-toploop:
- for acuri < numa && bcuri < numb {
- pass++
-
- isOverlap, isLeftoverA, isLeftoverB, leftoverstart, intersection = intersectWithLeftover32(astart, int64(a.iv[acuri].last), bstart, int64(b.iv[bcuri].last))
-
- if !isOverlap {
- switch {
- case astart < bstart:
- acuri, done = a.findNextIntervalThatIntersectsStartingFrom(acuri+1, bstart)
- if done {
- break toploop
- }
- astart = int64(a.iv[acuri].start)
-
- case astart > bstart:
- bcuri, done = b.findNextIntervalThatIntersectsStartingFrom(bcuri+1, astart)
- if done {
- break toploop
- }
- bstart = int64(b.iv[bcuri].start)
-
- //default:
- // panic("impossible that astart == bstart, since !isOverlap")
- }
-
- } else {
- // isOverlap
- output = append(output, intersection)
- switch {
- case isLeftoverA:
- // note that we change astart without advancing acuri,
- // since we need to capture any 2ndary intersections with a.iv[acuri]
- astart = leftoverstart
- bcuri++
- if bcuri >= numb {
- break toploop
- }
- bstart = int64(b.iv[bcuri].start)
- case isLeftoverB:
- // note that we change bstart without advancing bcuri,
- // since we need to capture any 2ndary intersections with b.iv[bcuri]
- bstart = leftoverstart
- acuri++
- if acuri >= numa {
- break toploop
- }
- astart = int64(a.iv[acuri].start)
- default:
- // neither had leftover, both completely consumed
- // optionally, assert for sanity:
- //if a.iv[acuri].endx != b.iv[bcuri].endx {
- // panic("huh? should only be possible that endx agree now!")
- //}
-
- // advance to next a interval
- acuri++
- if acuri >= numa {
- break toploop
- }
- astart = int64(a.iv[acuri].start)
-
- // advance to next b interval
- bcuri++
- if bcuri >= numb {
- break toploop
- }
- bstart = int64(b.iv[bcuri].start)
- }
- }
- } // end for toploop
-
- if len(output) == 0 {
- return res
- }
-
- res.iv = output
- return res
-}
-
-// intersectCardinality returns the cardinality of the
-// intersection of rc (also known as 'a') and b.
-func (rc *runContainer32) intersectCardinality(b *runContainer32) int64 {
- answer := int64(0)
-
- a := rc
- numa := int64(len(a.iv))
- numb := int64(len(b.iv))
- if numa == 0 || numb == 0 {
- return 0
- }
-
- if numa == 1 && numb == 1 {
- if !haveOverlap32(a.iv[0], b.iv[0]) {
- return 0
- }
- }
-
- var acuri int64
- var bcuri int64
-
- astart := int64(a.iv[acuri].start)
- bstart := int64(b.iv[bcuri].start)
-
- var intersection interval32
- var leftoverstart int64
- var isOverlap, isLeftoverA, isLeftoverB bool
- var done bool
- pass := 0
-toploop:
- for acuri < numa && bcuri < numb {
- pass++
-
- isOverlap, isLeftoverA, isLeftoverB, leftoverstart, intersection = intersectWithLeftover32(astart, int64(a.iv[acuri].last), bstart, int64(b.iv[bcuri].last))
-
- if !isOverlap {
- switch {
- case astart < bstart:
- acuri, done = a.findNextIntervalThatIntersectsStartingFrom(acuri+1, bstart)
- if done {
- break toploop
- }
- astart = int64(a.iv[acuri].start)
-
- case astart > bstart:
- bcuri, done = b.findNextIntervalThatIntersectsStartingFrom(bcuri+1, astart)
- if done {
- break toploop
- }
- bstart = int64(b.iv[bcuri].start)
-
- //default:
- // panic("impossible that astart == bstart, since !isOverlap")
- }
-
- } else {
- // isOverlap
- answer += int64(intersection.last) - int64(intersection.start) + 1
- switch {
- case isLeftoverA:
- // note that we change astart without advancing acuri,
- // since we need to capture any 2ndary intersections with a.iv[acuri]
- astart = leftoverstart
- bcuri++
- if bcuri >= numb {
- break toploop
- }
- bstart = int64(b.iv[bcuri].start)
- case isLeftoverB:
- // note that we change bstart without advancing bcuri,
- // since we need to capture any 2ndary intersections with b.iv[bcuri]
- bstart = leftoverstart
- acuri++
- if acuri >= numa {
- break toploop
- }
- astart = int64(a.iv[acuri].start)
- default:
- // neither had leftover, both completely consumed
- // optionally, assert for sanity:
- //if a.iv[acuri].endx != b.iv[bcuri].endx {
- // panic("huh? should only be possible that endx agree now!")
- //}
-
- // advance to next a interval
- acuri++
- if acuri >= numa {
- break toploop
- }
- astart = int64(a.iv[acuri].start)
-
- // advance to next b interval
- bcuri++
- if bcuri >= numb {
- break toploop
- }
- bstart = int64(b.iv[bcuri].start)
- }
- }
- } // end for toploop
-
- return answer
-}
-
-// get returns true if key is in the container.
-func (rc *runContainer32) contains(key uint32) bool {
- _, in, _ := rc.search(int64(key), nil)
- return in
-}
-
-// numIntervals returns the count of intervals in the container.
-func (rc *runContainer32) numIntervals() int {
- return len(rc.iv)
-}
-
-// search returns alreadyPresent to indicate if the
-// key is already in one of our interval32s.
-//
-// If key is alreadyPresent, then whichInterval32 tells
-// you where.
-//
-// If key is not already present, then whichInterval32 is
-// set as follows:
-//
-// a) whichInterval32 == len(rc.iv)-1 if key is beyond our
-// last interval32 in rc.iv;
-//
-// b) whichInterval32 == -1 if key is before our first
-// interval32 in rc.iv;
-//
-// c) whichInterval32 is set to the minimum index of rc.iv
-// which comes strictly before the key;
-// so rc.iv[whichInterval32].last < key,
-// and if whichInterval32+1 exists, then key < rc.iv[whichInterval32+1].start
-// (Note that whichInterval32+1 won't exist when
-// whichInterval32 is the last interval.)
-//
-// runContainer32.search always returns whichInterval32 < len(rc.iv).
-//
-// If not nil, opts can be used to further restrict
-// the search space.
-//
-func (rc *runContainer32) search(key int64, opts *searchOptions) (whichInterval32 int64, alreadyPresent bool, numCompares int) {
- n := int64(len(rc.iv))
- if n == 0 {
- return -1, false, 0
- }
-
- startIndex := int64(0)
- endxIndex := n
- if opts != nil {
- startIndex = opts.startIndex
-
- // let endxIndex == 0 mean no effect
- if opts.endxIndex > 0 {
- endxIndex = opts.endxIndex
- }
- }
-
- // sort.Search returns the smallest index i
- // in [0, n) at which f(i) is true, assuming that on the range [0, n),
- // f(i) == true implies f(i+1) == true.
- // If there is no such index, Search returns n.
-
- // For correctness, this began as verbatim snippet from
- // sort.Search in the Go standard lib.
- // We inline our comparison function for speed, and
- // annotate with numCompares
- // to observe and test that extra bounds are utilized.
- i, j := startIndex, endxIndex
- for i < j {
- h := i + (j-i)/2 // avoid overflow when computing h as the bisector
- // i <= h < j
- numCompares++
- if !(key < int64(rc.iv[h].start)) {
- i = h + 1
- } else {
- j = h
- }
- }
- below := i
- // end std lib snippet.
-
- // The above is a simple in-lining and annotation of:
- /* below := sort.Search(n,
- func(i int) bool {
- return key < rc.iv[i].start
- })
- */
- whichInterval32 = below - 1
-
- if below == n {
- // all falses => key is >= start of all interval32s
- // ... so does it belong to the last interval32?
- if key < int64(rc.iv[n-1].last)+1 {
- // yes, it belongs to the last interval32
- alreadyPresent = true
- return
- }
- // no, it is beyond the last interval32.
- // leave alreadyPreset = false
- return
- }
-
- // INVAR: key is below rc.iv[below]
- if below == 0 {
- // key is before the first first interval32.
- // leave alreadyPresent = false
- return
- }
-
- // INVAR: key is >= rc.iv[below-1].start and
- // key is < rc.iv[below].start
-
- // is key in below-1 interval32?
- if key >= int64(rc.iv[below-1].start) && key < int64(rc.iv[below-1].last)+1 {
- // yes, it is. key is in below-1 interval32.
- alreadyPresent = true
- return
- }
-
- // INVAR: key >= rc.iv[below-1].endx && key < rc.iv[below].start
- // leave alreadyPresent = false
- return
-}
-
-// cardinality returns the count of the integers stored in the
-// runContainer32.
-func (rc *runContainer32) cardinality() int64 {
- if len(rc.iv) == 0 {
- rc.card = 0
- return 0
- }
- if rc.card > 0 {
- return rc.card // already cached
- }
- // have to compute it
- var n int64
- for _, p := range rc.iv {
- n += p.runlen()
- }
- rc.card = n // cache it
- return n
-}
-
-// AsSlice decompresses the contents into a []uint32 slice.
-func (rc *runContainer32) AsSlice() []uint32 {
- s := make([]uint32, rc.cardinality())
- j := 0
- for _, p := range rc.iv {
- for i := p.start; i <= p.last; i++ {
- s[j] = i
- j++
- }
- }
- return s
-}
-
-// newRunContainer32 creates an empty run container.
-func newRunContainer32() *runContainer32 {
- return &runContainer32{}
-}
-
-// newRunContainer32CopyIv creates a run container, initializing
-// with a copy of the supplied iv slice.
-//
-func newRunContainer32CopyIv(iv []interval32) *runContainer32 {
- rc := &runContainer32{
- iv: make([]interval32, len(iv)),
- }
- copy(rc.iv, iv)
- return rc
-}
-
-func (rc *runContainer32) Clone() *runContainer32 {
- rc2 := newRunContainer32CopyIv(rc.iv)
- return rc2
-}
-
-// newRunContainer32TakeOwnership returns a new runContainer32
-// backed by the provided iv slice, which we will
-// assume exclusive control over from now on.
-//
-func newRunContainer32TakeOwnership(iv []interval32) *runContainer32 {
- rc := &runContainer32{
- iv: iv,
- }
- return rc
-}
-
-const baseRc32Size = int(unsafe.Sizeof(runContainer32{}))
-const perIntervalRc32Size = int(unsafe.Sizeof(interval32{}))
-
-const baseDiskRc32Size = int(unsafe.Sizeof(uint32(0)))
-
-// see also runContainer32SerializedSizeInBytes(numRuns int) int
-
-// getSizeInBytes returns the number of bytes of memory
-// required by this runContainer32.
-func (rc *runContainer32) getSizeInBytes() int {
- return perIntervalRc32Size*len(rc.iv) + baseRc32Size
-}
-
-// runContainer32SerializedSizeInBytes returns the number of bytes of disk
-// required to hold numRuns in a runContainer32.
-func runContainer32SerializedSizeInBytes(numRuns int) int {
- return perIntervalRc32Size*numRuns + baseDiskRc32Size
-}
-
-// Add adds a single value k to the set.
-func (rc *runContainer32) Add(k uint32) (wasNew bool) {
- // TODO comment from runContainer32.java:
- // it might be better and simpler to do return
- // toBitmapOrArrayContainer(getCardinality()).add(k)
- // but note that some unit tests use this method to build up test
- // runcontainers without calling runOptimize
-
- k64 := int64(k)
-
- index, present, _ := rc.search(k64, nil)
- if present {
- return // already there
- }
- wasNew = true
-
- // increment card if it is cached already
- if rc.card > 0 {
- rc.card++
- }
- n := int64(len(rc.iv))
- if index == -1 {
- // we may need to extend the first run
- if n > 0 {
- if rc.iv[0].start == k+1 {
- rc.iv[0].start = k
- return
- }
- }
- // nope, k stands alone, starting the new first interval32.
- rc.iv = append([]interval32{{start: k, last: k}}, rc.iv...)
- return
- }
-
- // are we off the end? handle both index == n and index == n-1:
- if index >= n-1 {
- if int64(rc.iv[n-1].last)+1 == k64 {
- rc.iv[n-1].last++
- return
- }
- rc.iv = append(rc.iv, interval32{start: k, last: k})
- return
- }
-
- // INVAR: index and index+1 both exist, and k goes between them.
- //
- // Now: add k into the middle,
- // possibly fusing with index or index+1 interval32
- // and possibly resulting in fusing of two interval32s
- // that had a one integer gap.
-
- left := index
- right := index + 1
-
- // are we fusing left and right by adding k?
- if int64(rc.iv[left].last)+1 == k64 && int64(rc.iv[right].start) == k64+1 {
- // fuse into left
- rc.iv[left].last = rc.iv[right].last
- // remove redundant right
- rc.iv = append(rc.iv[:left+1], rc.iv[right+1:]...)
- return
- }
-
- // are we an addition to left?
- if int64(rc.iv[left].last)+1 == k64 {
- // yes
- rc.iv[left].last++
- return
- }
-
- // are we an addition to right?
- if int64(rc.iv[right].start) == k64+1 {
- // yes
- rc.iv[right].start = k
- return
- }
-
- // k makes a standalone new interval32, inserted in the middle
- tail := append([]interval32{{start: k, last: k}}, rc.iv[right:]...)
- rc.iv = append(rc.iv[:left+1], tail...)
- return
-}
-
-//msgp:ignore runIterator
-
-// runIterator32 advice: you must call Next() at least once
-// before calling Cur(); and you should call HasNext()
-// before calling Next() to insure there are contents.
-type runIterator32 struct {
- rc *runContainer32
- curIndex int64
- curPosInIndex uint32
- curSeq int64
-}
-
-// newRunIterator32 returns a new empty run container.
-func (rc *runContainer32) newRunIterator32() *runIterator32 {
- return &runIterator32{rc: rc, curIndex: -1}
-}
-
-// HasNext returns false if calling Next will panic. It
-// returns true when there is at least one more value
-// available in the iteration sequence.
-func (ri *runIterator32) hasNext() bool {
- if len(ri.rc.iv) == 0 {
- return false
- }
- if ri.curIndex == -1 {
- return true
- }
- return ri.curSeq+1 < ri.rc.cardinality()
-}
-
-// cur returns the current value pointed to by the iterator.
-func (ri *runIterator32) cur() uint32 {
- return ri.rc.iv[ri.curIndex].start + ri.curPosInIndex
-}
-
-// Next returns the next value in the iteration sequence.
-func (ri *runIterator32) next() uint32 {
- if !ri.hasNext() {
- panic("no Next available")
- }
- if ri.curIndex >= int64(len(ri.rc.iv)) {
- panic("runIterator.Next() going beyond what is available")
- }
- if ri.curIndex == -1 {
- // first time is special
- ri.curIndex = 0
- } else {
- ri.curPosInIndex++
- if int64(ri.rc.iv[ri.curIndex].start)+int64(ri.curPosInIndex) == int64(ri.rc.iv[ri.curIndex].last)+1 {
- ri.curPosInIndex = 0
- ri.curIndex++
- }
- ri.curSeq++
- }
- return ri.cur()
-}
-
-// remove removes the element that the iterator
-// is on from the run container. You can use
-// Cur if you want to double check what is about
-// to be deleted.
-func (ri *runIterator32) remove() uint32 {
- n := ri.rc.cardinality()
- if n == 0 {
- panic("runIterator.Remove called on empty runContainer32")
- }
- cur := ri.cur()
-
- ri.rc.deleteAt(&ri.curIndex, &ri.curPosInIndex, &ri.curSeq)
- return cur
-}
-
-// remove removes key from the container.
-func (rc *runContainer32) removeKey(key uint32) (wasPresent bool) {
-
- var index int64
- var curSeq int64
- index, wasPresent, _ = rc.search(int64(key), nil)
- if !wasPresent {
- return // already removed, nothing to do.
- }
- pos := key - rc.iv[index].start
- rc.deleteAt(&index, &pos, &curSeq)
- return
-}
-
-// internal helper functions
-
-func (rc *runContainer32) deleteAt(curIndex *int64, curPosInIndex *uint32, curSeq *int64) {
- rc.card--
- (*curSeq)--
- ci := *curIndex
- pos := *curPosInIndex
-
- // are we first, last, or in the middle of our interval32?
- switch {
- case pos == 0:
- if int64(rc.iv[ci].start) == int64(rc.iv[ci].last) {
- // our interval disappears
- rc.iv = append(rc.iv[:ci], rc.iv[ci+1:]...)
- // curIndex stays the same, since the delete did
- // the advance for us.
- *curPosInIndex = 0
- } else {
- rc.iv[ci].start++ // no longer overflowable
- }
- case int64(pos) == rc.iv[ci].runlen()-1:
- // last
- rc.iv[ci].last--
- // our interval32 cannot disappear, else we would have been pos == 0, case first above.
- (*curPosInIndex)--
- // if we leave *curIndex alone, then Next() will work properly even after the delete.
- default:
- //middle
- // split into two, adding an interval32
- new0 := interval32{
- start: rc.iv[ci].start,
- last: rc.iv[ci].start + *curPosInIndex - 1}
-
- new1start := int64(rc.iv[ci].start) + int64(*curPosInIndex) + 1
- if new1start > int64(MaxUint32) {
- panic("overflow?!?!")
- }
- new1 := interval32{
- start: uint32(new1start),
- last: rc.iv[ci].last}
- tail := append([]interval32{new0, new1}, rc.iv[ci+1:]...)
- rc.iv = append(rc.iv[:ci], tail...)
- // update curIndex and curPosInIndex
- (*curIndex)++
- *curPosInIndex = 0
- }
-
-}
-
-func have4Overlap32(astart, alast, bstart, blast int64) bool {
- if alast+1 <= bstart {
- return false
- }
- return blast+1 > astart
-}
-
-func intersectWithLeftover32(astart, alast, bstart, blast int64) (isOverlap, isLeftoverA, isLeftoverB bool, leftoverstart int64, intersection interval32) {
- if !have4Overlap32(astart, alast, bstart, blast) {
- return
- }
- isOverlap = true
-
- // do the intersection:
- if bstart > astart {
- intersection.start = uint32(bstart)
- } else {
- intersection.start = uint32(astart)
- }
- switch {
- case blast < alast:
- isLeftoverA = true
- leftoverstart = blast + 1
- intersection.last = uint32(blast)
- case alast < blast:
- isLeftoverB = true
- leftoverstart = alast + 1
- intersection.last = uint32(alast)
- default:
- // alast == blast
- intersection.last = uint32(alast)
- }
-
- return
-}
-
-func (rc *runContainer32) findNextIntervalThatIntersectsStartingFrom(startIndex int64, key int64) (index int64, done bool) {
-
- rc.myOpts.startIndex = startIndex
- rc.myOpts.endxIndex = 0
-
- w, _, _ := rc.search(key, &rc.myOpts)
- // rc.search always returns w < len(rc.iv)
- if w < startIndex {
- // not found and comes before lower bound startIndex,
- // so just use the lower bound.
- if startIndex == int64(len(rc.iv)) {
- // also this bump up means that we are done
- return startIndex, true
- }
- return startIndex, false
- }
-
- return w, false
-}
-
-func sliceToString32(m []interval32) string {
- s := ""
- for i := range m {
- s += fmt.Sprintf("%v: %s, ", i, m[i])
- }
- return s
-}
-
-// selectInt32 returns the j-th value in the container.
-// We panic of j is out of bounds.
-func (rc *runContainer32) selectInt32(j uint32) int {
- n := rc.cardinality()
- if int64(j) > n {
- panic(fmt.Sprintf("Cannot select %v since Cardinality is %v", j, n))
- }
-
- var offset int64
- for k := range rc.iv {
- nextOffset := offset + rc.iv[k].runlen() + 1
- if nextOffset > int64(j) {
- return int(int64(rc.iv[k].start) + (int64(j) - offset))
- }
- offset = nextOffset
- }
- panic(fmt.Sprintf("Cannot select %v since Cardinality is %v", j, n))
-}
-
-// helper for invert
-func (rc *runContainer32) invertlastInterval(origin uint32, lastIdx int) []interval32 {
- cur := rc.iv[lastIdx]
- if cur.last == MaxUint32 {
- if cur.start == origin {
- return nil // empty container
- }
- return []interval32{{start: origin, last: cur.start - 1}}
- }
- if cur.start == origin {
- return []interval32{{start: cur.last + 1, last: MaxUint32}}
- }
- // invert splits
- return []interval32{
- {start: origin, last: cur.start - 1},
- {start: cur.last + 1, last: MaxUint32},
- }
-}
-
-// invert returns a new container (not inplace), that is
-// the inversion of rc. For each bit b in rc, the
-// returned value has !b
-func (rc *runContainer32) invert() *runContainer32 {
- ni := len(rc.iv)
- var m []interval32
- switch ni {
- case 0:
- return &runContainer32{iv: []interval32{{0, MaxUint32}}}
- case 1:
- return &runContainer32{iv: rc.invertlastInterval(0, 0)}
- }
- var invstart int64
- ult := ni - 1
- for i, cur := range rc.iv {
- if i == ult {
- // invertlastInteval will add both intervals (b) and (c) in
- // diagram below.
- m = append(m, rc.invertlastInterval(uint32(invstart), i)...)
- break
- }
- // INVAR: i and cur are not the last interval, there is a next at i+1
- //
- // ........[cur.start, cur.last] ...... [next.start, next.last]....
- // ^ ^ ^
- // (a) (b) (c)
- //
- // Now: we add interval (a); but if (a) is empty, for cur.start==0, we skip it.
- if cur.start > 0 {
- m = append(m, interval32{start: uint32(invstart), last: cur.start - 1})
- }
- invstart = int64(cur.last + 1)
- }
- return &runContainer32{iv: m}
-}
-
-func (iv interval32) equal(b interval32) bool {
- if iv.start == b.start {
- return iv.last == b.last
- }
- return false
-}
-
-func (iv interval32) isSuperSetOf(b interval32) bool {
- return iv.start <= b.start && b.last <= iv.last
-}
-
-func (iv interval32) subtractInterval(del interval32) (left []interval32, delcount int64) {
- isect, isEmpty := intersectInterval32s(iv, del)
-
- if isEmpty {
- return nil, 0
- }
- if del.isSuperSetOf(iv) {
- return nil, iv.runlen()
- }
-
- switch {
- case isect.start > iv.start && isect.last < iv.last:
- new0 := interval32{start: iv.start, last: isect.start - 1}
- new1 := interval32{start: isect.last + 1, last: iv.last}
- return []interval32{new0, new1}, isect.runlen()
- case isect.start == iv.start:
- return []interval32{{start: isect.last + 1, last: iv.last}}, isect.runlen()
- default:
- return []interval32{{start: iv.start, last: isect.start - 1}}, isect.runlen()
- }
-}
-
-func (rc *runContainer32) isubtract(del interval32) {
- origiv := make([]interval32, len(rc.iv))
- copy(origiv, rc.iv)
- n := int64(len(rc.iv))
- if n == 0 {
- return // already done.
- }
-
- _, isEmpty := intersectInterval32s(
- interval32{
- start: rc.iv[0].start,
- last: rc.iv[n-1].last,
- }, del)
- if isEmpty {
- return // done
- }
- // INVAR there is some intersection between rc and del
- istart, startAlready, _ := rc.search(int64(del.start), nil)
- ilast, lastAlready, _ := rc.search(int64(del.last), nil)
- rc.card = -1
- if istart == -1 {
- if ilast == n-1 && !lastAlready {
- rc.iv = nil
- return
- }
- }
- // some intervals will remain
- switch {
- case startAlready && lastAlready:
- res0, _ := rc.iv[istart].subtractInterval(del)
-
- // would overwrite values in iv b/c res0 can have len 2. so
- // write to origiv instead.
- lost := 1 + ilast - istart
- changeSize := int64(len(res0)) - lost
- newSize := int64(len(rc.iv)) + changeSize
-
- // rc.iv = append(pre, caboose...)
- // return
-
- if ilast != istart {
- res1, _ := rc.iv[ilast].subtractInterval(del)
- res0 = append(res0, res1...)
- changeSize = int64(len(res0)) - lost
- newSize = int64(len(rc.iv)) + changeSize
- }
- switch {
- case changeSize < 0:
- // shrink
- copy(rc.iv[istart+int64(len(res0)):], rc.iv[ilast+1:])
- copy(rc.iv[istart:istart+int64(len(res0))], res0)
- rc.iv = rc.iv[:newSize]
- return
- case changeSize == 0:
- // stay the same
- copy(rc.iv[istart:istart+int64(len(res0))], res0)
- return
- default:
- // changeSize > 0 is only possible when ilast == istart.
- // Hence we now know: changeSize == 1 and len(res0) == 2
- rc.iv = append(rc.iv, interval32{})
- // len(rc.iv) is correct now, no need to rc.iv = rc.iv[:newSize]
-
- // copy the tail into place
- copy(rc.iv[ilast+2:], rc.iv[ilast+1:])
- // copy the new item(s) into place
- copy(rc.iv[istart:istart+2], res0)
- return
- }
-
- case !startAlready && !lastAlready:
- // we get to discard whole intervals
-
- // from the search() definition:
-
- // if del.start is not present, then istart is
- // set as follows:
- //
- // a) istart == n-1 if del.start is beyond our
- // last interval32 in rc.iv;
- //
- // b) istart == -1 if del.start is before our first
- // interval32 in rc.iv;
- //
- // c) istart is set to the minimum index of rc.iv
- // which comes strictly before the del.start;
- // so del.start > rc.iv[istart].last,
- // and if istart+1 exists, then del.start < rc.iv[istart+1].startx
-
- // if del.last is not present, then ilast is
- // set as follows:
- //
- // a) ilast == n-1 if del.last is beyond our
- // last interval32 in rc.iv;
- //
- // b) ilast == -1 if del.last is before our first
- // interval32 in rc.iv;
- //
- // c) ilast is set to the minimum index of rc.iv
- // which comes strictly before the del.last;
- // so del.last > rc.iv[ilast].last,
- // and if ilast+1 exists, then del.last < rc.iv[ilast+1].start
-
- // INVAR: istart >= 0
- pre := rc.iv[:istart+1]
- if ilast == n-1 {
- rc.iv = pre
- return
- }
- // INVAR: ilast < n-1
- lost := ilast - istart
- changeSize := -lost
- newSize := int64(len(rc.iv)) + changeSize
- if changeSize != 0 {
- copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:])
- }
- rc.iv = rc.iv[:newSize]
- return
-
- case startAlready && !lastAlready:
- // we can only shrink or stay the same size
- // i.e. we either eliminate the whole interval,
- // or just cut off the right side.
- res0, _ := rc.iv[istart].subtractInterval(del)
- if len(res0) > 0 {
- // len(res) must be 1
- rc.iv[istart] = res0[0]
- }
- lost := 1 + (ilast - istart)
- changeSize := int64(len(res0)) - lost
- newSize := int64(len(rc.iv)) + changeSize
- if changeSize != 0 {
- copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:])
- }
- rc.iv = rc.iv[:newSize]
- return
-
- case !startAlready && lastAlready:
- // we can only shrink or stay the same size
- res1, _ := rc.iv[ilast].subtractInterval(del)
- lost := ilast - istart
- changeSize := int64(len(res1)) - lost
- newSize := int64(len(rc.iv)) + changeSize
- if changeSize != 0 {
- // move the tail first to make room for res1
- copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:])
- }
- copy(rc.iv[istart+1:], res1)
- rc.iv = rc.iv[:newSize]
- return
- }
-}
-
-// compute rc minus b, and return the result as a new value (not inplace).
-// port of run_container_andnot from CRoaring...
-// https://github.com/RoaringBitmap/CRoaring/blob/master/src/containers/run.c#L435-L496
-func (rc *runContainer32) AndNotRunContainer32(b *runContainer32) *runContainer32 {
-
- if len(b.iv) == 0 || len(rc.iv) == 0 {
- return rc
- }
-
- dst := newRunContainer32()
- apos := 0
- bpos := 0
-
- a := rc
-
- astart := a.iv[apos].start
- alast := a.iv[apos].last
- bstart := b.iv[bpos].start
- blast := b.iv[bpos].last
-
- alen := len(a.iv)
- blen := len(b.iv)
-
- for apos < alen && bpos < blen {
- switch {
- case alast < bstart:
- // output the first run
- dst.iv = append(dst.iv, interval32{start: astart, last: alast})
- apos++
- if apos < alen {
- astart = a.iv[apos].start
- alast = a.iv[apos].last
- }
- case blast < astart:
- // exit the second run
- bpos++
- if bpos < blen {
- bstart = b.iv[bpos].start
- blast = b.iv[bpos].last
- }
- default:
- // a: [ ]
- // b: [ ]
- // alast >= bstart
- // blast >= astart
- if astart < bstart {
- dst.iv = append(dst.iv, interval32{start: astart, last: bstart - 1})
- }
- if alast > blast {
- astart = blast + 1
- } else {
- apos++
- if apos < alen {
- astart = a.iv[apos].start
- alast = a.iv[apos].last
- }
- }
- }
- }
- if apos < alen {
- dst.iv = append(dst.iv, interval32{start: astart, last: alast})
- apos++
- if apos < alen {
- dst.iv = append(dst.iv, a.iv[apos:]...)
- }
- }
-
- return dst
-}
-
-func (rc *runContainer32) numberOfRuns() (nr int) {
- return len(rc.iv)
-}
-
-func (rc *runContainer32) containerType() contype {
- return run32Contype
-}
-
-func (rc *runContainer32) equals32(srb *runContainer32) bool {
- //p("both rc32")
- // Check if the containers are the same object.
- if rc == srb {
- //p("same object")
- return true
- }
-
- if len(srb.iv) != len(rc.iv) {
- //p("iv len differ")
- return false
- }
-
- for i, v := range rc.iv {
- if v != srb.iv[i] {
- //p("differ at iv i=%v, srb.iv[i]=%v, rc.iv[i]=%v", i, srb.iv[i], rc.iv[i])
- return false
- }
- }
- //p("all intervals same, returning true")
- return true
-}
diff --git a/vendor/github.com/RoaringBitmap/roaring/rle_gen.go b/vendor/github.com/RoaringBitmap/roaring/rle_gen.go
deleted file mode 100644
index bc9da75f3a..0000000000
--- a/vendor/github.com/RoaringBitmap/roaring/rle_gen.go
+++ /dev/null
@@ -1,1118 +0,0 @@
-package roaring
-
-// NOTE: THIS FILE WAS PRODUCED BY THE
-// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
-// DO NOT EDIT
-
-import "github.com/tinylib/msgp/msgp"
-
-// DecodeMsg implements msgp.Decodable
-func (z *addHelper32) DecodeMsg(dc *msgp.Reader) (err error) {
- var field []byte
- _ = field
- var zbai uint32
- zbai, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zbai > 0 {
- zbai--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "runstart":
- z.runstart, err = dc.ReadUint32()
- if err != nil {
- return
- }
- case "runlen":
- z.runlen, err = dc.ReadUint32()
- if err != nil {
- return
- }
- case "actuallyAdded":
- z.actuallyAdded, err = dc.ReadUint32()
- if err != nil {
- return
- }
- case "m":
- var zcmr uint32
- zcmr, err = dc.ReadArrayHeader()
- if err != nil {
- return
- }
- if cap(z.m) >= int(zcmr) {
- z.m = (z.m)[:zcmr]
- } else {
- z.m = make([]interval32, zcmr)
- }
- for zxvk := range z.m {
- var zajw uint32
- zajw, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zajw > 0 {
- zajw--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "start":
- z.m[zxvk].start, err = dc.ReadUint32()
- if err != nil {
- return
- }
- case "last":
- z.m[zxvk].last, err = dc.ReadUint32()
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- }
- case "rc":
- if dc.IsNil() {
- err = dc.ReadNil()
- if err != nil {
- return
- }
- z.rc = nil
- } else {
- if z.rc == nil {
- z.rc = new(runContainer32)
- }
- var zwht uint32
- zwht, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zwht > 0 {
- zwht--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "iv":
- var zhct uint32
- zhct, err = dc.ReadArrayHeader()
- if err != nil {
- return
- }
- if cap(z.rc.iv) >= int(zhct) {
- z.rc.iv = (z.rc.iv)[:zhct]
- } else {
- z.rc.iv = make([]interval32, zhct)
- }
- for zbzg := range z.rc.iv {
- var zcua uint32
- zcua, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zcua > 0 {
- zcua--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "start":
- z.rc.iv[zbzg].start, err = dc.ReadUint32()
- if err != nil {
- return
- }
- case "last":
- z.rc.iv[zbzg].last, err = dc.ReadUint32()
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- }
- case "card":
- z.rc.card, err = dc.ReadInt64()
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z *addHelper32) EncodeMsg(en *msgp.Writer) (err error) {
- // map header, size 5
- // write "runstart"
- err = en.Append(0x85, 0xa8, 0x72, 0x75, 0x6e, 0x73, 0x74, 0x61, 0x72, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint32(z.runstart)
- if err != nil {
- return
- }
- // write "runlen"
- err = en.Append(0xa6, 0x72, 0x75, 0x6e, 0x6c, 0x65, 0x6e)
- if err != nil {
- return err
- }
- err = en.WriteUint32(z.runlen)
- if err != nil {
- return
- }
- // write "actuallyAdded"
- err = en.Append(0xad, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x6c, 0x79, 0x41, 0x64, 0x64, 0x65, 0x64)
- if err != nil {
- return err
- }
- err = en.WriteUint32(z.actuallyAdded)
- if err != nil {
- return
- }
- // write "m"
- err = en.Append(0xa1, 0x6d)
- if err != nil {
- return err
- }
- err = en.WriteArrayHeader(uint32(len(z.m)))
- if err != nil {
- return
- }
- for zxvk := range z.m {
- // map header, size 2
- // write "start"
- err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint32(z.m[zxvk].start)
- if err != nil {
- return
- }
- // write "last"
- err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint32(z.m[zxvk].last)
- if err != nil {
- return
- }
- }
- // write "rc"
- err = en.Append(0xa2, 0x72, 0x63)
- if err != nil {
- return err
- }
- if z.rc == nil {
- err = en.WriteNil()
- if err != nil {
- return
- }
- } else {
- // map header, size 2
- // write "iv"
- err = en.Append(0x82, 0xa2, 0x69, 0x76)
- if err != nil {
- return err
- }
- err = en.WriteArrayHeader(uint32(len(z.rc.iv)))
- if err != nil {
- return
- }
- for zbzg := range z.rc.iv {
- // map header, size 2
- // write "start"
- err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint32(z.rc.iv[zbzg].start)
- if err != nil {
- return
- }
- // write "last"
- err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint32(z.rc.iv[zbzg].last)
- if err != nil {
- return
- }
- }
- // write "card"
- err = en.Append(0xa4, 0x63, 0x61, 0x72, 0x64)
- if err != nil {
- return err
- }
- err = en.WriteInt64(z.rc.card)
- if err != nil {
- return
- }
- }
- return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z *addHelper32) MarshalMsg(b []byte) (o []byte, err error) {
- o = msgp.Require(b, z.Msgsize())
- // map header, size 5
- // string "runstart"
- o = append(o, 0x85, 0xa8, 0x72, 0x75, 0x6e, 0x73, 0x74, 0x61, 0x72, 0x74)
- o = msgp.AppendUint32(o, z.runstart)
- // string "runlen"
- o = append(o, 0xa6, 0x72, 0x75, 0x6e, 0x6c, 0x65, 0x6e)
- o = msgp.AppendUint32(o, z.runlen)
- // string "actuallyAdded"
- o = append(o, 0xad, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x6c, 0x79, 0x41, 0x64, 0x64, 0x65, 0x64)
- o = msgp.AppendUint32(o, z.actuallyAdded)
- // string "m"
- o = append(o, 0xa1, 0x6d)
- o = msgp.AppendArrayHeader(o, uint32(len(z.m)))
- for zxvk := range z.m {
- // map header, size 2
- // string "start"
- o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
- o = msgp.AppendUint32(o, z.m[zxvk].start)
- // string "last"
- o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74)
- o = msgp.AppendUint32(o, z.m[zxvk].last)
- }
- // string "rc"
- o = append(o, 0xa2, 0x72, 0x63)
- if z.rc == nil {
- o = msgp.AppendNil(o)
- } else {
- // map header, size 2
- // string "iv"
- o = append(o, 0x82, 0xa2, 0x69, 0x76)
- o = msgp.AppendArrayHeader(o, uint32(len(z.rc.iv)))
- for zbzg := range z.rc.iv {
- // map header, size 2
- // string "start"
- o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
- o = msgp.AppendUint32(o, z.rc.iv[zbzg].start)
- // string "last"
- o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74)
- o = msgp.AppendUint32(o, z.rc.iv[zbzg].last)
- }
- // string "card"
- o = append(o, 0xa4, 0x63, 0x61, 0x72, 0x64)
- o = msgp.AppendInt64(o, z.rc.card)
- }
- return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *addHelper32) UnmarshalMsg(bts []byte) (o []byte, err error) {
- var field []byte
- _ = field
- var zxhx uint32
- zxhx, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zxhx > 0 {
- zxhx--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "runstart":
- z.runstart, bts, err = msgp.ReadUint32Bytes(bts)
- if err != nil {
- return
- }
- case "runlen":
- z.runlen, bts, err = msgp.ReadUint32Bytes(bts)
- if err != nil {
- return
- }
- case "actuallyAdded":
- z.actuallyAdded, bts, err = msgp.ReadUint32Bytes(bts)
- if err != nil {
- return
- }
- case "m":
- var zlqf uint32
- zlqf, bts, err = msgp.ReadArrayHeaderBytes(bts)
- if err != nil {
- return
- }
- if cap(z.m) >= int(zlqf) {
- z.m = (z.m)[:zlqf]
- } else {
- z.m = make([]interval32, zlqf)
- }
- for zxvk := range z.m {
- var zdaf uint32
- zdaf, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zdaf > 0 {
- zdaf--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "start":
- z.m[zxvk].start, bts, err = msgp.ReadUint32Bytes(bts)
- if err != nil {
- return
- }
- case "last":
- z.m[zxvk].last, bts, err = msgp.ReadUint32Bytes(bts)
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- }
- case "rc":
- if msgp.IsNil(bts) {
- bts, err = msgp.ReadNilBytes(bts)
- if err != nil {
- return
- }
- z.rc = nil
- } else {
- if z.rc == nil {
- z.rc = new(runContainer32)
- }
- var zpks uint32
- zpks, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zpks > 0 {
- zpks--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "iv":
- var zjfb uint32
- zjfb, bts, err = msgp.ReadArrayHeaderBytes(bts)
- if err != nil {
- return
- }
- if cap(z.rc.iv) >= int(zjfb) {
- z.rc.iv = (z.rc.iv)[:zjfb]
- } else {
- z.rc.iv = make([]interval32, zjfb)
- }
- for zbzg := range z.rc.iv {
- var zcxo uint32
- zcxo, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zcxo > 0 {
- zcxo--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "start":
- z.rc.iv[zbzg].start, bts, err = msgp.ReadUint32Bytes(bts)
- if err != nil {
- return
- }
- case "last":
- z.rc.iv[zbzg].last, bts, err = msgp.ReadUint32Bytes(bts)
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- }
- case "card":
- z.rc.card, bts, err = msgp.ReadInt64Bytes(bts)
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- o = bts
- return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *addHelper32) Msgsize() (s int) {
- s = 1 + 9 + msgp.Uint32Size + 7 + msgp.Uint32Size + 14 + msgp.Uint32Size + 2 + msgp.ArrayHeaderSize + (len(z.m) * (12 + msgp.Uint32Size + msgp.Uint32Size)) + 3
- if z.rc == nil {
- s += msgp.NilSize
- } else {
- s += 1 + 3 + msgp.ArrayHeaderSize + (len(z.rc.iv) * (12 + msgp.Uint32Size + msgp.Uint32Size)) + 5 + msgp.Int64Size
- }
- return
-}
-
-// DecodeMsg implements msgp.Decodable
-func (z *interval32) DecodeMsg(dc *msgp.Reader) (err error) {
- var field []byte
- _ = field
- var zeff uint32
- zeff, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zeff > 0 {
- zeff--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "start":
- z.start, err = dc.ReadUint32()
- if err != nil {
- return
- }
- case "last":
- z.last, err = dc.ReadUint32()
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z interval32) EncodeMsg(en *msgp.Writer) (err error) {
- // map header, size 2
- // write "start"
- err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint32(z.start)
- if err != nil {
- return
- }
- // write "last"
- err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint32(z.last)
- if err != nil {
- return
- }
- return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z interval32) MarshalMsg(b []byte) (o []byte, err error) {
- o = msgp.Require(b, z.Msgsize())
- // map header, size 2
- // string "start"
- o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
- o = msgp.AppendUint32(o, z.start)
- // string "last"
- o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74)
- o = msgp.AppendUint32(o, z.last)
- return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *interval32) UnmarshalMsg(bts []byte) (o []byte, err error) {
- var field []byte
- _ = field
- var zrsw uint32
- zrsw, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zrsw > 0 {
- zrsw--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "start":
- z.start, bts, err = msgp.ReadUint32Bytes(bts)
- if err != nil {
- return
- }
- case "last":
- z.last, bts, err = msgp.ReadUint32Bytes(bts)
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- o = bts
- return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z interval32) Msgsize() (s int) {
- s = 1 + 6 + msgp.Uint32Size + 5 + msgp.Uint32Size
- return
-}
-
-// DecodeMsg implements msgp.Decodable
-func (z *runContainer32) DecodeMsg(dc *msgp.Reader) (err error) {
- var field []byte
- _ = field
- var zdnj uint32
- zdnj, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zdnj > 0 {
- zdnj--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "iv":
- var zobc uint32
- zobc, err = dc.ReadArrayHeader()
- if err != nil {
- return
- }
- if cap(z.iv) >= int(zobc) {
- z.iv = (z.iv)[:zobc]
- } else {
- z.iv = make([]interval32, zobc)
- }
- for zxpk := range z.iv {
- var zsnv uint32
- zsnv, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zsnv > 0 {
- zsnv--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "start":
- z.iv[zxpk].start, err = dc.ReadUint32()
- if err != nil {
- return
- }
- case "last":
- z.iv[zxpk].last, err = dc.ReadUint32()
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- }
- case "card":
- z.card, err = dc.ReadInt64()
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z *runContainer32) EncodeMsg(en *msgp.Writer) (err error) {
- // map header, size 2
- // write "iv"
- err = en.Append(0x82, 0xa2, 0x69, 0x76)
- if err != nil {
- return err
- }
- err = en.WriteArrayHeader(uint32(len(z.iv)))
- if err != nil {
- return
- }
- for zxpk := range z.iv {
- // map header, size 2
- // write "start"
- err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint32(z.iv[zxpk].start)
- if err != nil {
- return
- }
- // write "last"
- err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint32(z.iv[zxpk].last)
- if err != nil {
- return
- }
- }
- // write "card"
- err = en.Append(0xa4, 0x63, 0x61, 0x72, 0x64)
- if err != nil {
- return err
- }
- err = en.WriteInt64(z.card)
- if err != nil {
- return
- }
- return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z *runContainer32) MarshalMsg(b []byte) (o []byte, err error) {
- o = msgp.Require(b, z.Msgsize())
- // map header, size 2
- // string "iv"
- o = append(o, 0x82, 0xa2, 0x69, 0x76)
- o = msgp.AppendArrayHeader(o, uint32(len(z.iv)))
- for zxpk := range z.iv {
- // map header, size 2
- // string "start"
- o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
- o = msgp.AppendUint32(o, z.iv[zxpk].start)
- // string "last"
- o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74)
- o = msgp.AppendUint32(o, z.iv[zxpk].last)
- }
- // string "card"
- o = append(o, 0xa4, 0x63, 0x61, 0x72, 0x64)
- o = msgp.AppendInt64(o, z.card)
- return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *runContainer32) UnmarshalMsg(bts []byte) (o []byte, err error) {
- var field []byte
- _ = field
- var zkgt uint32
- zkgt, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zkgt > 0 {
- zkgt--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "iv":
- var zema uint32
- zema, bts, err = msgp.ReadArrayHeaderBytes(bts)
- if err != nil {
- return
- }
- if cap(z.iv) >= int(zema) {
- z.iv = (z.iv)[:zema]
- } else {
- z.iv = make([]interval32, zema)
- }
- for zxpk := range z.iv {
- var zpez uint32
- zpez, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zpez > 0 {
- zpez--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "start":
- z.iv[zxpk].start, bts, err = msgp.ReadUint32Bytes(bts)
- if err != nil {
- return
- }
- case "last":
- z.iv[zxpk].last, bts, err = msgp.ReadUint32Bytes(bts)
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- }
- case "card":
- z.card, bts, err = msgp.ReadInt64Bytes(bts)
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- o = bts
- return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *runContainer32) Msgsize() (s int) {
- s = 1 + 3 + msgp.ArrayHeaderSize + (len(z.iv) * (12 + msgp.Uint32Size + msgp.Uint32Size)) + 5 + msgp.Int64Size
- return
-}
-
-// DecodeMsg implements msgp.Decodable
-func (z *runIterator32) DecodeMsg(dc *msgp.Reader) (err error) {
- var field []byte
- _ = field
- var zqke uint32
- zqke, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zqke > 0 {
- zqke--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "rc":
- if dc.IsNil() {
- err = dc.ReadNil()
- if err != nil {
- return
- }
- z.rc = nil
- } else {
- if z.rc == nil {
- z.rc = new(runContainer32)
- }
- err = z.rc.DecodeMsg(dc)
- if err != nil {
- return
- }
- }
- case "curIndex":
- z.curIndex, err = dc.ReadInt64()
- if err != nil {
- return
- }
- case "curPosInIndex":
- z.curPosInIndex, err = dc.ReadUint32()
- if err != nil {
- return
- }
- case "curSeq":
- z.curSeq, err = dc.ReadInt64()
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z *runIterator32) EncodeMsg(en *msgp.Writer) (err error) {
- // map header, size 4
- // write "rc"
- err = en.Append(0x84, 0xa2, 0x72, 0x63)
- if err != nil {
- return err
- }
- if z.rc == nil {
- err = en.WriteNil()
- if err != nil {
- return
- }
- } else {
- err = z.rc.EncodeMsg(en)
- if err != nil {
- return
- }
- }
- // write "curIndex"
- err = en.Append(0xa8, 0x63, 0x75, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78)
- if err != nil {
- return err
- }
- err = en.WriteInt64(z.curIndex)
- if err != nil {
- return
- }
- // write "curPosInIndex"
- err = en.Append(0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78)
- if err != nil {
- return err
- }
- err = en.WriteUint32(z.curPosInIndex)
- if err != nil {
- return
- }
- // write "curSeq"
- err = en.Append(0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71)
- if err != nil {
- return err
- }
- err = en.WriteInt64(z.curSeq)
- if err != nil {
- return
- }
- return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z *runIterator32) MarshalMsg(b []byte) (o []byte, err error) {
- o = msgp.Require(b, z.Msgsize())
- // map header, size 4
- // string "rc"
- o = append(o, 0x84, 0xa2, 0x72, 0x63)
- if z.rc == nil {
- o = msgp.AppendNil(o)
- } else {
- o, err = z.rc.MarshalMsg(o)
- if err != nil {
- return
- }
- }
- // string "curIndex"
- o = append(o, 0xa8, 0x63, 0x75, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78)
- o = msgp.AppendInt64(o, z.curIndex)
- // string "curPosInIndex"
- o = append(o, 0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78)
- o = msgp.AppendUint32(o, z.curPosInIndex)
- // string "curSeq"
- o = append(o, 0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71)
- o = msgp.AppendInt64(o, z.curSeq)
- return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *runIterator32) UnmarshalMsg(bts []byte) (o []byte, err error) {
- var field []byte
- _ = field
- var zqyh uint32
- zqyh, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zqyh > 0 {
- zqyh--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "rc":
- if msgp.IsNil(bts) {
- bts, err = msgp.ReadNilBytes(bts)
- if err != nil {
- return
- }
- z.rc = nil
- } else {
- if z.rc == nil {
- z.rc = new(runContainer32)
- }
- bts, err = z.rc.UnmarshalMsg(bts)
- if err != nil {
- return
- }
- }
- case "curIndex":
- z.curIndex, bts, err = msgp.ReadInt64Bytes(bts)
- if err != nil {
- return
- }
- case "curPosInIndex":
- z.curPosInIndex, bts, err = msgp.ReadUint32Bytes(bts)
- if err != nil {
- return
- }
- case "curSeq":
- z.curSeq, bts, err = msgp.ReadInt64Bytes(bts)
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- o = bts
- return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *runIterator32) Msgsize() (s int) {
- s = 1 + 3
- if z.rc == nil {
- s += msgp.NilSize
- } else {
- s += z.rc.Msgsize()
- }
- s += 9 + msgp.Int64Size + 14 + msgp.Uint32Size + 7 + msgp.Int64Size
- return
-}
-
-// DecodeMsg implements msgp.Decodable
-func (z *uint32Slice) DecodeMsg(dc *msgp.Reader) (err error) {
- var zjpj uint32
- zjpj, err = dc.ReadArrayHeader()
- if err != nil {
- return
- }
- if cap((*z)) >= int(zjpj) {
- (*z) = (*z)[:zjpj]
- } else {
- (*z) = make(uint32Slice, zjpj)
- }
- for zywj := range *z {
- (*z)[zywj], err = dc.ReadUint32()
- if err != nil {
- return
- }
- }
- return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z uint32Slice) EncodeMsg(en *msgp.Writer) (err error) {
- err = en.WriteArrayHeader(uint32(len(z)))
- if err != nil {
- return
- }
- for zzpf := range z {
- err = en.WriteUint32(z[zzpf])
- if err != nil {
- return
- }
- }
- return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z uint32Slice) MarshalMsg(b []byte) (o []byte, err error) {
- o = msgp.Require(b, z.Msgsize())
- o = msgp.AppendArrayHeader(o, uint32(len(z)))
- for zzpf := range z {
- o = msgp.AppendUint32(o, z[zzpf])
- }
- return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *uint32Slice) UnmarshalMsg(bts []byte) (o []byte, err error) {
- var zgmo uint32
- zgmo, bts, err = msgp.ReadArrayHeaderBytes(bts)
- if err != nil {
- return
- }
- if cap((*z)) >= int(zgmo) {
- (*z) = (*z)[:zgmo]
- } else {
- (*z) = make(uint32Slice, zgmo)
- }
- for zrfe := range *z {
- (*z)[zrfe], bts, err = msgp.ReadUint32Bytes(bts)
- if err != nil {
- return
- }
- }
- o = bts
- return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z uint32Slice) Msgsize() (s int) {
- s = msgp.ArrayHeaderSize + (len(z) * (msgp.Uint32Size))
- return
-}
diff --git a/vendor/github.com/RoaringBitmap/roaring/rlecommon.go b/vendor/github.com/RoaringBitmap/roaring/rlecommon.go
deleted file mode 100644
index 133636787a..0000000000
--- a/vendor/github.com/RoaringBitmap/roaring/rlecommon.go
+++ /dev/null
@@ -1,163 +0,0 @@
-package roaring
-
-import (
- "fmt"
-)
-
-// common to rle32.go and rle16.go
-
-// rleVerbose controls whether p() prints show up.
-// The testing package sets this based on
-// testing.Verbose().
-var rleVerbose bool
-
-// p is a shorthand for fmt.Printf with beginning and
-// trailing newlines. p() makes it easy
-// to add diagnostic print statements.
-func p(format string, args ...interface{}) {
- if rleVerbose {
- fmt.Printf("\n"+format+"\n", args...)
- }
-}
-
-// MaxUint32 is the largest uint32 value.
-const MaxUint32 = 4294967295
-
-// MaxUint16 is the largest 16 bit unsigned int.
-// This is the largest value an interval16 can store.
-const MaxUint16 = 65535
-
-// searchOptions allows us to accelerate runContainer32.search with
-// prior knowledge of (mostly lower) bounds. This is used by Union
-// and Intersect.
-type searchOptions struct {
- // start here instead of at 0
- startIndex int64
-
- // upper bound instead of len(rc.iv);
- // endxIndex == 0 means ignore the bound and use
- // endxIndex == n ==len(rc.iv) which is also
- // naturally the default for search()
- // when opt = nil.
- endxIndex int64
-}
-
-// And finds the intersection of rc and b.
-func (rc *runContainer32) And(b *Bitmap) *Bitmap {
- out := NewBitmap()
- for _, p := range rc.iv {
- for i := p.start; i <= p.last; i++ {
- if b.Contains(i) {
- out.Add(i)
- }
- }
- }
- return out
-}
-
-// Xor returns the exclusive-or of rc and b.
-func (rc *runContainer32) Xor(b *Bitmap) *Bitmap {
- out := b.Clone()
- for _, p := range rc.iv {
- for v := p.start; v <= p.last; v++ {
- if out.Contains(v) {
- out.RemoveRange(uint64(v), uint64(v+1))
- } else {
- out.Add(v)
- }
- }
- }
- return out
-}
-
-// Or returns the union of rc and b.
-func (rc *runContainer32) Or(b *Bitmap) *Bitmap {
- out := b.Clone()
- for _, p := range rc.iv {
- for v := p.start; v <= p.last; v++ {
- out.Add(v)
- }
- }
- return out
-}
-
-// trial is used in the randomized testing of runContainers
-type trial struct {
- n int
- percentFill float64
- ntrial int
-
- // only in the union test
- // only subtract test
- percentDelete float64
-
- // only in 067 randomized operations
- // we do this + 1 passes
- numRandomOpsPass int
-
- // allow sampling range control
- // only recent tests respect this.
- srang *interval16
-}
-
-// And finds the intersection of rc and b.
-func (rc *runContainer16) And(b *Bitmap) *Bitmap {
- out := NewBitmap()
- for _, p := range rc.iv {
- plast := p.last()
- for i := p.start; i <= plast; i++ {
- if b.Contains(uint32(i)) {
- out.Add(uint32(i))
- }
- }
- }
- return out
-}
-
-// Xor returns the exclusive-or of rc and b.
-func (rc *runContainer16) Xor(b *Bitmap) *Bitmap {
- out := b.Clone()
- for _, p := range rc.iv {
- plast := p.last()
- for v := p.start; v <= plast; v++ {
- w := uint32(v)
- if out.Contains(w) {
- out.RemoveRange(uint64(w), uint64(w+1))
- } else {
- out.Add(w)
- }
- }
- }
- return out
-}
-
-// Or returns the union of rc and b.
-func (rc *runContainer16) Or(b *Bitmap) *Bitmap {
- out := b.Clone()
- for _, p := range rc.iv {
- plast := p.last()
- for v := p.start; v <= plast; v++ {
- out.Add(uint32(v))
- }
- }
- return out
-}
-
-//func (rc *runContainer32) and(container) container {
-// panic("TODO. not yet implemented")
-//}
-
-// serializedSizeInBytes returns the number of bytes of memory
-// required by this runContainer16. This is for the
-// Roaring format, as specified https://github.com/RoaringBitmap/RoaringFormatSpec/
-func (rc *runContainer16) serializedSizeInBytes() int {
- // number of runs in one uint16, then each run
- // needs two more uint16
- return 2 + len(rc.iv)*4
-}
-
-// serializedSizeInBytes returns the number of bytes of memory
-// required by this runContainer32.
-func (rc *runContainer32) serializedSizeInBytes() int {
- return 4 + len(rc.iv)*8
-}
diff --git a/vendor/github.com/RoaringBitmap/roaring/rlei.go b/vendor/github.com/RoaringBitmap/roaring/rlei.go
deleted file mode 100644
index a15a017e47..0000000000
--- a/vendor/github.com/RoaringBitmap/roaring/rlei.go
+++ /dev/null
@@ -1,695 +0,0 @@
-package roaring
-
-///////////////////////////////////////////////////
-//
-// container interface methods for runContainer16
-//
-///////////////////////////////////////////////////
-
-import (
- "fmt"
-)
-
-// compile time verify we meet interface requirements
-var _ container = &runContainer16{}
-
-func (rc *runContainer16) clone() container {
- return newRunContainer16CopyIv(rc.iv)
-}
-
-func (rc *runContainer16) minimum() uint16 {
- return rc.iv[0].start // assume not empty
-}
-
-func (rc *runContainer16) maximum() uint16 {
- return rc.iv[len(rc.iv)-1].last() // assume not empty
-}
-
-func (rc *runContainer16) isFull() bool {
- return (len(rc.iv) == 1) && ((rc.iv[0].start == 0) && (rc.iv[0].last() == MaxUint16))
-}
-
-func (rc *runContainer16) and(a container) container {
- if rc.isFull() {
- return a.clone()
- }
- switch c := a.(type) {
- case *runContainer16:
- return rc.intersect(c)
- case *arrayContainer:
- return rc.andArray(c)
- case *bitmapContainer:
- return rc.andBitmapContainer(c)
- }
- panic("unsupported container type")
-}
-
-func (rc *runContainer16) andCardinality(a container) int {
- switch c := a.(type) {
- case *runContainer16:
- return int(rc.intersectCardinality(c))
- case *arrayContainer:
- return rc.andArrayCardinality(c)
- case *bitmapContainer:
- return rc.andBitmapContainerCardinality(c)
- }
- panic("unsupported container type")
-}
-
-// andBitmapContainer finds the intersection of rc and b.
-func (rc *runContainer16) andBitmapContainer(bc *bitmapContainer) container {
- bc2 := newBitmapContainerFromRun(rc)
- return bc2.andBitmap(bc)
-}
-
-func (rc *runContainer16) andArrayCardinality(ac *arrayContainer) int {
- pos := 0
- answer := 0
- maxpos := ac.getCardinality()
- if maxpos == 0 {
- return 0 // won't happen in actual code
- }
- v := ac.content[pos]
-mainloop:
- for _, p := range rc.iv {
- for v < p.start {
- pos++
- if pos == maxpos {
- break mainloop
- }
- v = ac.content[pos]
- }
- for v <= p.last() {
- answer++
- pos++
- if pos == maxpos {
- break mainloop
- }
- v = ac.content[pos]
- }
- }
- return answer
-}
-
-func (rc *runContainer16) iand(a container) container {
- if rc.isFull() {
- return a.clone()
- }
- switch c := a.(type) {
- case *runContainer16:
- return rc.inplaceIntersect(c)
- case *arrayContainer:
- return rc.andArray(c)
- case *bitmapContainer:
- return rc.iandBitmapContainer(c)
- }
- panic("unsupported container type")
-}
-
-func (rc *runContainer16) inplaceIntersect(rc2 *runContainer16) container {
- // TODO: optimize by doing less allocation, possibly?
-
- // sect will be new
- sect := rc.intersect(rc2)
- *rc = *sect
- return rc
-}
-
-func (rc *runContainer16) iandBitmapContainer(bc *bitmapContainer) container {
- isect := rc.andBitmapContainer(bc)
- *rc = *newRunContainer16FromContainer(isect)
- return rc
-}
-
-func (rc *runContainer16) andArray(ac *arrayContainer) container {
- if len(rc.iv) == 0 {
- return newArrayContainer()
- }
-
- acCardinality := ac.getCardinality()
- c := newArrayContainerCapacity(acCardinality)
-
- for rlePos, arrayPos := 0, 0; arrayPos < acCardinality; {
- iv := rc.iv[rlePos]
- arrayVal := ac.content[arrayPos]
-
- for iv.last() < arrayVal {
- rlePos++
- if rlePos == len(rc.iv) {
- return c
- }
- iv = rc.iv[rlePos]
- }
-
- if iv.start > arrayVal {
- arrayPos = advanceUntil(ac.content, arrayPos, len(ac.content), iv.start)
- } else {
- c.content = append(c.content, arrayVal)
- arrayPos++
- }
- }
- return c
-}
-
-func (rc *runContainer16) andNot(a container) container {
- switch c := a.(type) {
- case *arrayContainer:
- return rc.andNotArray(c)
- case *bitmapContainer:
- return rc.andNotBitmap(c)
- case *runContainer16:
- return rc.andNotRunContainer16(c)
- }
- panic("unsupported container type")
-}
-
-func (rc *runContainer16) fillLeastSignificant16bits(x []uint32, i int, mask uint32) {
- k := 0
- var val int64
- for _, p := range rc.iv {
- n := p.runlen()
- for j := int64(0); j < n; j++ {
- val = int64(p.start) + j
- x[k+i] = uint32(val) | mask
- k++
- }
- }
-}
-
-func (rc *runContainer16) getShortIterator() shortIterable {
- return rc.newRunIterator16()
-}
-
-func (rc *runContainer16) getManyIterator() manyIterable {
- return rc.newManyRunIterator16()
-}
-
-// add the values in the range [firstOfRange, endx). endx
-// is still abe to express 2^16 because it is an int not an uint16.
-func (rc *runContainer16) iaddRange(firstOfRange, endx int) container {
-
- if firstOfRange >= endx {
- panic(fmt.Sprintf("invalid %v = endx >= firstOfRange", endx))
- }
- addme := newRunContainer16TakeOwnership([]interval16{
- {
- start: uint16(firstOfRange),
- length: uint16(endx - 1 - firstOfRange),
- },
- })
- *rc = *rc.union(addme)
- return rc
-}
-
-// remove the values in the range [firstOfRange,endx)
-func (rc *runContainer16) iremoveRange(firstOfRange, endx int) container {
- if firstOfRange >= endx {
- panic(fmt.Sprintf("request to iremove empty set [%v, %v),"+
- " nothing to do.", firstOfRange, endx))
- //return rc
- }
- x := newInterval16Range(uint16(firstOfRange), uint16(endx-1))
- rc.isubtract(x)
- return rc
-}
-
-// not flip the values in the range [firstOfRange,endx)
-func (rc *runContainer16) not(firstOfRange, endx int) container {
- if firstOfRange >= endx {
- panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange))
- }
-
- return rc.Not(firstOfRange, endx)
-}
-
-// Not flips the values in the range [firstOfRange,endx).
-// This is not inplace. Only the returned value has the flipped bits.
-//
-// Currently implemented as (!A intersect B) union (A minus B),
-// where A is rc, and B is the supplied [firstOfRange, endx) interval.
-//
-// TODO(time optimization): convert this to a single pass
-// algorithm by copying AndNotRunContainer16() and modifying it.
-// Current routine is correct but
-// makes 2 more passes through the arrays than should be
-// strictly necessary. Measure both ways though--this may not matter.
-//
-func (rc *runContainer16) Not(firstOfRange, endx int) *runContainer16 {
-
- if firstOfRange >= endx {
- panic(fmt.Sprintf("invalid %v = endx >= firstOfRange == %v", endx, firstOfRange))
- }
-
- if firstOfRange >= endx {
- return rc.Clone()
- }
-
- a := rc
- // algo:
- // (!A intersect B) union (A minus B)
-
- nota := a.invert()
-
- bs := []interval16{newInterval16Range(uint16(firstOfRange), uint16(endx-1))}
- b := newRunContainer16TakeOwnership(bs)
-
- notAintersectB := nota.intersect(b)
-
- aMinusB := a.AndNotRunContainer16(b)
-
- rc2 := notAintersectB.union(aMinusB)
- return rc2
-}
-
-// equals is now logical equals; it does not require the
-// same underlying container type.
-func (rc *runContainer16) equals(o container) bool {
- srb, ok := o.(*runContainer16)
-
- if !ok {
- // maybe value instead of pointer
- val, valok := o.(*runContainer16)
- if valok {
- srb = val
- ok = true
- }
- }
- if ok {
- // Check if the containers are the same object.
- if rc == srb {
- return true
- }
-
- if len(srb.iv) != len(rc.iv) {
- return false
- }
-
- for i, v := range rc.iv {
- if v != srb.iv[i] {
- return false
- }
- }
- return true
- }
-
- // use generic comparison
- if o.getCardinality() != rc.getCardinality() {
- return false
- }
- rit := rc.getShortIterator()
- bit := o.getShortIterator()
-
- //k := 0
- for rit.hasNext() {
- if bit.next() != rit.next() {
- return false
- }
- //k++
- }
- return true
-}
-
-func (rc *runContainer16) iaddReturnMinimized(x uint16) container {
- rc.Add(x)
- return rc
-}
-
-func (rc *runContainer16) iadd(x uint16) (wasNew bool) {
- return rc.Add(x)
-}
-
-func (rc *runContainer16) iremoveReturnMinimized(x uint16) container {
- rc.removeKey(x)
- return rc
-}
-
-func (rc *runContainer16) iremove(x uint16) bool {
- return rc.removeKey(x)
-}
-
-func (rc *runContainer16) or(a container) container {
- if rc.isFull() {
- return rc.clone()
- }
- switch c := a.(type) {
- case *runContainer16:
- return rc.union(c)
- case *arrayContainer:
- return rc.orArray(c)
- case *bitmapContainer:
- return rc.orBitmapContainer(c)
- }
- panic("unsupported container type")
-}
-
-func (rc *runContainer16) orCardinality(a container) int {
- switch c := a.(type) {
- case *runContainer16:
- return int(rc.unionCardinality(c))
- case *arrayContainer:
- return rc.orArrayCardinality(c)
- case *bitmapContainer:
- return rc.orBitmapContainerCardinality(c)
- }
- panic("unsupported container type")
-}
-
-// orBitmapContainer finds the union of rc and bc.
-func (rc *runContainer16) orBitmapContainer(bc *bitmapContainer) container {
- bc2 := newBitmapContainerFromRun(rc)
- return bc2.iorBitmap(bc)
-}
-
-func (rc *runContainer16) andBitmapContainerCardinality(bc *bitmapContainer) int {
- answer := 0
- for i := range rc.iv {
- answer += bc.getCardinalityInRange(uint(rc.iv[i].start), uint(rc.iv[i].last())+1)
- }
- //bc.computeCardinality()
- return answer
-}
-
-func (rc *runContainer16) orBitmapContainerCardinality(bc *bitmapContainer) int {
- return rc.getCardinality() + bc.getCardinality() - rc.andBitmapContainerCardinality(bc)
-}
-
-// orArray finds the union of rc and ac.
-func (rc *runContainer16) orArray(ac *arrayContainer) container {
- bc1 := newBitmapContainerFromRun(rc)
- bc2 := ac.toBitmapContainer()
- return bc1.orBitmap(bc2)
-}
-
-// orArray finds the union of rc and ac.
-func (rc *runContainer16) orArrayCardinality(ac *arrayContainer) int {
- return ac.getCardinality() + rc.getCardinality() - rc.andArrayCardinality(ac)
-}
-
-func (rc *runContainer16) ior(a container) container {
- if rc.isFull() {
- return rc
- }
- switch c := a.(type) {
- case *runContainer16:
- return rc.inplaceUnion(c)
- case *arrayContainer:
- return rc.iorArray(c)
- case *bitmapContainer:
- return rc.iorBitmapContainer(c)
- }
- panic("unsupported container type")
-}
-
-func (rc *runContainer16) inplaceUnion(rc2 *runContainer16) container {
- p("rc.inplaceUnion with len(rc2.iv)=%v", len(rc2.iv))
- for _, p := range rc2.iv {
- last := int64(p.last())
- for i := int64(p.start); i <= last; i++ {
- rc.Add(uint16(i))
- }
- }
- return rc
-}
-
-func (rc *runContainer16) iorBitmapContainer(bc *bitmapContainer) container {
-
- it := bc.getShortIterator()
- for it.hasNext() {
- rc.Add(it.next())
- }
- return rc
-}
-
-func (rc *runContainer16) iorArray(ac *arrayContainer) container {
- it := ac.getShortIterator()
- for it.hasNext() {
- rc.Add(it.next())
- }
- return rc
-}
-
-// lazyIOR is described (not yet implemented) in
-// this nice note from @lemire on
-// https://github.com/RoaringBitmap/roaring/pull/70#issuecomment-263613737
-//
-// Description of lazyOR and lazyIOR from @lemire:
-//
-// Lazy functions are optional and can be simply
-// wrapper around non-lazy functions.
-//
-// The idea of "laziness" is as follows. It is
-// inspired by the concept of lazy evaluation
-// you might be familiar with (functional programming
-// and all that). So a roaring bitmap is
-// such that all its containers are, in some
-// sense, chosen to use as little memory as
-// possible. This is nice. Also, all bitsets
-// are "cardinality aware" so that you can do
-// fast rank/select queries, or query the
-// cardinality of the whole bitmap... very fast,
-// without latency.
-//
-// However, imagine that you are aggregating 100
-// bitmaps together. So you OR the first two, then OR
-// that with the third one and so forth. Clearly,
-// intermediate bitmaps don't need to be as
-// compressed as possible, right? They can be
-// in a "dirty state". You only need the end
-// result to be in a nice state... which you
-// can achieve by calling repairAfterLazy at the end.
-//
-// The Java/C code does something special for
-// the in-place lazy OR runs. The idea is that
-// instead of taking two run containers and
-// generating a new one, we actually try to
-// do the computation in-place through a
-// technique invented by @gssiyankai (pinging him!).
-// What you do is you check whether the host
-// run container has lots of extra capacity.
-// If it does, you move its data at the end of
-// the backing array, and then you write
-// the answer at the beginning. What this
-// trick does is minimize memory allocations.
-//
-func (rc *runContainer16) lazyIOR(a container) container {
- // not lazy at the moment
- // TODO: make it lazy
- return rc.ior(a)
-
- /*
- switch c := a.(type) {
- case *arrayContainer:
- return rc.lazyIorArray(c)
- case *bitmapContainer:
- return rc.lazyIorBitmap(c)
- case *runContainer16:
- return rc.lazyIorRun16(c)
- }
- panic("unsupported container type")
- */
-}
-
-// lazyOR is described above in lazyIOR.
-func (rc *runContainer16) lazyOR(a container) container {
-
- // not lazy at the moment
- // TODO: make it lazy
- return rc.or(a)
-
- /*
- switch c := a.(type) {
- case *arrayContainer:
- return rc.lazyOrArray(c)
- case *bitmapContainer:
- return rc.lazyOrBitmap(c)
- case *runContainer16:
- return rc.lazyOrRunContainer16(c)
- }
- panic("unsupported container type")
- */
-}
-
-func (rc *runContainer16) intersects(a container) bool {
- // TODO: optimize by doing inplace/less allocation, possibly?
- isect := rc.and(a)
- return isect.getCardinality() > 0
-}
-
-func (rc *runContainer16) xor(a container) container {
- switch c := a.(type) {
- case *arrayContainer:
- return rc.xorArray(c)
- case *bitmapContainer:
- return rc.xorBitmap(c)
- case *runContainer16:
- return rc.xorRunContainer16(c)
- }
- panic("unsupported container type")
-}
-
-func (rc *runContainer16) iandNot(a container) container {
- switch c := a.(type) {
- case *arrayContainer:
- return rc.iandNotArray(c)
- case *bitmapContainer:
- return rc.iandNotBitmap(c)
- case *runContainer16:
- return rc.iandNotRunContainer16(c)
- }
- panic("unsupported container type")
-}
-
-// flip the values in the range [firstOfRange,endx)
-func (rc *runContainer16) inot(firstOfRange, endx int) container {
- if firstOfRange >= endx {
- panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange))
- }
- // TODO: minimize copies, do it all inplace; not() makes a copy.
- rc = rc.Not(firstOfRange, endx)
- return rc
-}
-
-func (rc *runContainer16) getCardinality() int {
- return int(rc.cardinality())
-}
-
-func (rc *runContainer16) rank(x uint16) int {
- n := int64(len(rc.iv))
- xx := int64(x)
- w, already, _ := rc.search(xx, nil)
- if w < 0 {
- return 0
- }
- if !already && w == n-1 {
- return rc.getCardinality()
- }
- var rnk int64
- if !already {
- for i := int64(0); i <= w; i++ {
- rnk += rc.iv[i].runlen()
- }
- return int(rnk)
- }
- for i := int64(0); i < w; i++ {
- rnk += rc.iv[i].runlen()
- }
- rnk += int64(x-rc.iv[w].start) + 1
- return int(rnk)
-}
-
-func (rc *runContainer16) selectInt(x uint16) int {
- return rc.selectInt16(x)
-}
-
-func (rc *runContainer16) andNotRunContainer16(b *runContainer16) container {
- return rc.AndNotRunContainer16(b)
-}
-
-func (rc *runContainer16) andNotArray(ac *arrayContainer) container {
- rcb := rc.toBitmapContainer()
- acb := ac.toBitmapContainer()
- return rcb.andNotBitmap(acb)
-}
-
-func (rc *runContainer16) andNotBitmap(bc *bitmapContainer) container {
- rcb := rc.toBitmapContainer()
- return rcb.andNotBitmap(bc)
-}
-
-func (rc *runContainer16) toBitmapContainer() *bitmapContainer {
- p("run16 toBitmap starting; rc has %v ranges", len(rc.iv))
- bc := newBitmapContainer()
- for i := range rc.iv {
- bc.iaddRange(int(rc.iv[i].start), int(rc.iv[i].last())+1)
- }
- bc.computeCardinality()
- return bc
-}
-
-func (rc *runContainer16) iandNotRunContainer16(x2 *runContainer16) container {
- rcb := rc.toBitmapContainer()
- x2b := x2.toBitmapContainer()
- rcb.iandNotBitmapSurely(x2b)
- // TODO: check size and optimize the return value
- // TODO: is inplace modification really required? If not, elide the copy.
- rc2 := newRunContainer16FromBitmapContainer(rcb)
- *rc = *rc2
- return rc
-}
-
-func (rc *runContainer16) iandNotArray(ac *arrayContainer) container {
- rcb := rc.toBitmapContainer()
- acb := ac.toBitmapContainer()
- rcb.iandNotBitmapSurely(acb)
- // TODO: check size and optimize the return value
- // TODO: is inplace modification really required? If not, elide the copy.
- rc2 := newRunContainer16FromBitmapContainer(rcb)
- *rc = *rc2
- return rc
-}
-
-func (rc *runContainer16) iandNotBitmap(bc *bitmapContainer) container {
- rcb := rc.toBitmapContainer()
- rcb.iandNotBitmapSurely(bc)
- // TODO: check size and optimize the return value
- // TODO: is inplace modification really required? If not, elide the copy.
- rc2 := newRunContainer16FromBitmapContainer(rcb)
- *rc = *rc2
- return rc
-}
-
-func (rc *runContainer16) xorRunContainer16(x2 *runContainer16) container {
- rcb := rc.toBitmapContainer()
- x2b := x2.toBitmapContainer()
- return rcb.xorBitmap(x2b)
-}
-
-func (rc *runContainer16) xorArray(ac *arrayContainer) container {
- rcb := rc.toBitmapContainer()
- acb := ac.toBitmapContainer()
- return rcb.xorBitmap(acb)
-}
-
-func (rc *runContainer16) xorBitmap(bc *bitmapContainer) container {
- rcb := rc.toBitmapContainer()
- return rcb.xorBitmap(bc)
-}
-
-// convert to bitmap or array *if needed*
-func (rc *runContainer16) toEfficientContainer() container {
-
- // runContainer16SerializedSizeInBytes(numRuns)
- sizeAsRunContainer := rc.getSizeInBytes()
- sizeAsBitmapContainer := bitmapContainerSizeInBytes()
- card := int(rc.cardinality())
- sizeAsArrayContainer := arrayContainerSizeInBytes(card)
- if sizeAsRunContainer <= minOfInt(sizeAsBitmapContainer, sizeAsArrayContainer) {
- return rc
- }
- if card <= arrayDefaultMaxSize {
- return rc.toArrayContainer()
- }
- bc := newBitmapContainerFromRun(rc)
- return bc
-}
-
-func (rc *runContainer16) toArrayContainer() *arrayContainer {
- ac := newArrayContainer()
- for i := range rc.iv {
- ac.iaddRange(int(rc.iv[i].start), int(rc.iv[i].last())+1)
- }
- return ac
-}
-
-func newRunContainer16FromContainer(c container) *runContainer16 {
-
- switch x := c.(type) {
- case *runContainer16:
- return x.Clone()
- case *arrayContainer:
- return newRunContainer16FromArray(x)
- case *bitmapContainer:
- return newRunContainer16FromBitmapContainer(x)
- }
- panic("unsupported container type")
-}
diff --git a/vendor/github.com/RoaringBitmap/roaring/roaring.go b/vendor/github.com/RoaringBitmap/roaring/roaring.go
index 5045a41933..df58cc30b5 100644
--- a/vendor/github.com/RoaringBitmap/roaring/roaring.go
+++ b/vendor/github.com/RoaringBitmap/roaring/roaring.go
@@ -6,12 +6,12 @@
package roaring
import (
- "bufio"
"bytes"
"encoding/base64"
"fmt"
"io"
"strconv"
+ "sync"
)
// Bitmap represents a compressed bitmap where you can add integers.
@@ -52,7 +52,7 @@ func (rb *Bitmap) ToBytes() ([]byte, error) {
return rb.highlowcontainer.toBytes()
}
-// WriteToMsgpack writes a msgpack2/snappy-streaming compressed serialized
+// Deprecated: WriteToMsgpack writes a msgpack2/snappy-streaming compressed serialized
// version of this bitmap to stream. The format is not
// compatible with the WriteTo() format, and is
// experimental: it may produce smaller on disk
@@ -67,8 +67,14 @@ func (rb *Bitmap) WriteToMsgpack(stream io.Writer) (int64, error) {
// The format is compatible with other RoaringBitmap
// implementations (Java, C) and is documented here:
// https://github.com/RoaringBitmap/RoaringFormatSpec
-func (rb *Bitmap) ReadFrom(stream io.Reader) (int64, error) {
- return rb.highlowcontainer.readFrom(stream)
+func (rb *Bitmap) ReadFrom(reader io.Reader) (p int64, err error) {
+ stream := byteInputAdapterPool.Get().(*byteInputAdapter)
+ stream.reset(reader)
+
+ p, err = rb.highlowcontainer.readFrom(stream)
+ byteInputAdapterPool.Put(stream)
+
+ return
}
// FromBuffer creates a bitmap from its serialized version stored in buffer
@@ -87,10 +93,36 @@ func (rb *Bitmap) ReadFrom(stream io.Reader) (int64, error) {
// You should *not* change the copy-on-write status of the resulting
// bitmaps (SetCopyOnWrite).
//
-func (rb *Bitmap) FromBuffer(buf []byte) (int64, error) {
- return rb.highlowcontainer.fromBuffer(buf)
+// If buf becomes unavailable, then a bitmap created with
+// FromBuffer would be effectively broken. Furthermore, any
+// bitmap derived from this bitmap (e.g., via Or, And) might
+// also be broken. Thus, before making buf unavailable, you should
+// call CloneCopyOnWriteContainers on all such bitmaps.
+//
+func (rb *Bitmap) FromBuffer(buf []byte) (p int64, err error) {
+ stream := byteBufferPool.Get().(*byteBuffer)
+ stream.reset(buf)
+
+ p, err = rb.highlowcontainer.readFrom(stream)
+ byteBufferPool.Put(stream)
+
+ return
}
+var (
+ byteBufferPool = sync.Pool{
+ New: func() interface{} {
+ return &byteBuffer{}
+ },
+ }
+
+ byteInputAdapterPool = sync.Pool{
+ New: func() interface{} {
+ return &byteInputAdapter{}
+ },
+ }
+)
+
// RunOptimize attempts to further compress the runs of consecutive values found in the bitmap
func (rb *Bitmap) RunOptimize() {
rb.highlowcontainer.runOptimize()
@@ -101,7 +133,7 @@ func (rb *Bitmap) HasRunCompression() bool {
return rb.highlowcontainer.hasRunCompression()
}
-// ReadFromMsgpack reads a msgpack2/snappy-streaming serialized
+// Deprecated: ReadFromMsgpack reads a msgpack2/snappy-streaming serialized
// version of this bitmap from stream. The format is
// expected is that written by the WriteToMsgpack()
// call; see additional notes there.
@@ -110,29 +142,15 @@ func (rb *Bitmap) ReadFromMsgpack(stream io.Reader) (int64, error) {
}
// MarshalBinary implements the encoding.BinaryMarshaler interface for the bitmap
+// (same as ToBytes)
func (rb *Bitmap) MarshalBinary() ([]byte, error) {
- var buf bytes.Buffer
- writer := bufio.NewWriter(&buf)
- _, err := rb.WriteTo(writer)
- if err != nil {
- return nil, err
- }
- err = writer.Flush()
- if err != nil {
- return nil, err
- }
- return buf.Bytes(), nil
+ return rb.ToBytes()
}
// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface for the bitmap
func (rb *Bitmap) UnmarshalBinary(data []byte) error {
- var buf bytes.Buffer
- _, err := buf.Write(data)
- if err != nil {
- return err
- }
- reader := bufio.NewReader(&buf)
- _, err = rb.ReadFrom(reader)
+ r := bytes.NewReader(data)
+ _, err := rb.ReadFrom(r)
return err
}
@@ -215,10 +233,20 @@ type IntIterable interface {
Next() uint32
}
+// IntPeekable allows you to look at the next value without advancing and
+// advance as long as the next value is smaller than minval
+type IntPeekable interface {
+ IntIterable
+ // PeekNext peeks the next value without advancing the iterator
+ PeekNext() uint32
+ // AdvanceIfNeeded advances as long as the next value is smaller than minval
+ AdvanceIfNeeded(minval uint32)
+}
+
type intIterator struct {
pos int
hs uint32
- iter shortIterable
+ iter shortPeekable
highlowcontainer *roaringArray
}
@@ -244,6 +272,30 @@ func (ii *intIterator) Next() uint32 {
return x
}
+// PeekNext peeks the next value without advancing the iterator
+func (ii *intIterator) PeekNext() uint32 {
+ return uint32(ii.iter.peekNext()&maxLowBit) | ii.hs
+}
+
+// AdvanceIfNeeded advances as long as the next value is smaller than minval
+func (ii *intIterator) AdvanceIfNeeded(minval uint32) {
+ to := minval >> 16
+
+ for ii.HasNext() && (ii.hs>>16) < to {
+ ii.pos++
+ ii.init()
+ }
+
+ if ii.HasNext() && (ii.hs>>16) == to {
+ ii.iter.advanceIfNeeded(lowbits(minval))
+
+ if !ii.iter.hasNext() {
+ ii.pos++
+ ii.init()
+ }
+ }
+}
+
func newIntIterator(a *Bitmap) *intIterator {
p := new(intIterator)
p.pos = 0
@@ -252,6 +304,45 @@ func newIntIterator(a *Bitmap) *intIterator {
return p
}
+type intReverseIterator struct {
+ pos int
+ hs uint32
+ iter shortIterable
+ highlowcontainer *roaringArray
+}
+
+// HasNext returns true if there are more integers to iterate over
+func (ii *intReverseIterator) HasNext() bool {
+ return ii.pos >= 0
+}
+
+func (ii *intReverseIterator) init() {
+ if ii.pos >= 0 {
+ ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getReverseIterator()
+ ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16
+ } else {
+ ii.iter = nil
+ }
+}
+
+// Next returns the next integer
+func (ii *intReverseIterator) Next() uint32 {
+ x := uint32(ii.iter.next()) | ii.hs
+ if !ii.iter.hasNext() {
+ ii.pos = ii.pos - 1
+ ii.init()
+ }
+ return x
+}
+
+func newIntReverseIterator(a *Bitmap) *intReverseIterator {
+ p := new(intReverseIterator)
+ p.highlowcontainer = &a.highlowcontainer
+ p.pos = a.highlowcontainer.size() - 1
+ p.init()
+ return p
+}
+
// ManyIntIterable allows you to iterate over the values in a Bitmap
type ManyIntIterable interface {
// pass in a buffer to fill up with values, returns how many values were returned
@@ -325,12 +416,20 @@ func (rb *Bitmap) String() string {
return buffer.String()
}
-// Iterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order
-func (rb *Bitmap) Iterator() IntIterable {
+// Iterator creates a new IntPeekable to iterate over the integers contained in the bitmap, in sorted order;
+// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove).
+func (rb *Bitmap) Iterator() IntPeekable {
return newIntIterator(rb)
}
-// Iterator creates a new ManyIntIterable to iterate over the integers contained in the bitmap, in sorted order
+// ReverseIterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order;
+// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove).
+func (rb *Bitmap) ReverseIterator() IntIterable {
+ return newIntReverseIterator(rb)
+}
+
+// ManyIterator creates a new ManyIntIterable to iterate over the integers contained in the bitmap, in sorted order;
+// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove).
func (rb *Bitmap) ManyIterator() ManyIntIterable {
return newManyIntIterator(rb)
}
@@ -374,6 +473,46 @@ func (rb *Bitmap) Equals(o interface{}) bool {
return false
}
+// AddOffset adds the value 'offset' to each and every value in a bitmap, generating a new bitmap in the process
+func AddOffset(x *Bitmap, offset uint32) (answer *Bitmap) {
+ containerOffset := highbits(offset)
+ inOffset := lowbits(offset)
+ if inOffset == 0 {
+ answer = x.Clone()
+ for pos := 0; pos < answer.highlowcontainer.size(); pos++ {
+ key := answer.highlowcontainer.getKeyAtIndex(pos)
+ key += containerOffset
+ answer.highlowcontainer.keys[pos] = key
+ }
+ } else {
+ answer = New()
+ for pos := 0; pos < x.highlowcontainer.size(); pos++ {
+ key := x.highlowcontainer.getKeyAtIndex(pos)
+ key += containerOffset
+ c := x.highlowcontainer.getContainerAtIndex(pos)
+ offsetted := c.addOffset(inOffset)
+ if offsetted[0].getCardinality() > 0 {
+ curSize := answer.highlowcontainer.size()
+ lastkey := uint16(0)
+ if curSize > 0 {
+ lastkey = answer.highlowcontainer.getKeyAtIndex(curSize - 1)
+ }
+ if curSize > 0 && lastkey == key {
+ prev := answer.highlowcontainer.getContainerAtIndex(curSize - 1)
+ orrseult := prev.ior(offsetted[0])
+ answer.highlowcontainer.setContainerAtIndex(curSize-1, orrseult)
+ } else {
+ answer.highlowcontainer.appendContainer(key, offsetted[0], false)
+ }
+ }
+ if offsetted[1].getCardinality() > 0 {
+ answer.highlowcontainer.appendContainer(key+1, offsetted[1], false)
+ }
+ }
+ }
+ return answer
+}
+
// Add the integer x to the bitmap
func (rb *Bitmap) Add(x uint32) {
hb := highbits(x)
@@ -794,11 +933,6 @@ main:
}
}
-/*func (rb *Bitmap) Or(x2 *Bitmap) {
- results := Or(rb, x2) // Todo: could be computed in-place for reduced memory usage
- rb.highlowcontainer = results.highlowcontainer
-}*/
-
// AndNot computes the difference between two bitmaps and stores the result in the current bitmap
func (rb *Bitmap) AndNot(x2 *Bitmap) {
pos1 := 0
@@ -1086,10 +1220,10 @@ func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) {
return
}
- hbStart := highbits(uint32(rangeStart))
- lbStart := lowbits(uint32(rangeStart))
- hbLast := highbits(uint32(rangeEnd - 1))
- lbLast := lowbits(uint32(rangeEnd - 1))
+ hbStart := uint32(highbits(uint32(rangeStart)))
+ lbStart := uint32(lowbits(uint32(rangeStart)))
+ hbLast := uint32(highbits(uint32(rangeEnd - 1)))
+ lbLast := uint32(lowbits(uint32(rangeEnd - 1)))
var max uint32 = maxLowBit
for hb := hbStart; hb <= hbLast; hb++ {
@@ -1102,7 +1236,7 @@ func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) {
containerLast = uint32(lbLast)
}
- i := rb.highlowcontainer.getIndex(hb)
+ i := rb.highlowcontainer.getIndex(uint16(hb))
if i >= 0 {
c := rb.highlowcontainer.getWritableContainerAtIndex(i).inot(int(containerStart), int(containerLast)+1)
@@ -1113,7 +1247,7 @@ func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) {
}
} else { // *think* the range of ones must never be
// empty.
- rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, rangeOfOnes(int(containerStart), int(containerLast)))
+ rb.highlowcontainer.insertNewKeyValueAt(-i-1, uint16(hb), rangeOfOnes(int(containerStart), int(containerLast)))
}
}
}
@@ -1139,24 +1273,24 @@ func (rb *Bitmap) AddRange(rangeStart, rangeEnd uint64) {
lbLast := uint32(lowbits(uint32(rangeEnd - 1)))
var max uint32 = maxLowBit
- for hb := uint16(hbStart); hb <= uint16(hbLast); hb++ {
+ for hb := hbStart; hb <= hbLast; hb++ {
containerStart := uint32(0)
- if hb == uint16(hbStart) {
+ if hb == hbStart {
containerStart = lbStart
}
containerLast := max
- if hb == uint16(hbLast) {
+ if hb == hbLast {
containerLast = lbLast
}
- i := rb.highlowcontainer.getIndex(hb)
+ i := rb.highlowcontainer.getIndex(uint16(hb))
if i >= 0 {
c := rb.highlowcontainer.getWritableContainerAtIndex(i).iaddRange(int(containerStart), int(containerLast)+1)
rb.highlowcontainer.setContainerAtIndex(i, c)
} else { // *think* the range of ones must never be
// empty.
- rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, rangeOfOnes(int(containerStart), int(containerLast)))
+ rb.highlowcontainer.insertNewKeyValueAt(-i-1, uint16(hb), rangeOfOnes(int(containerStart), int(containerLast)))
}
}
}
@@ -1243,13 +1377,13 @@ func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap {
}
answer := NewBitmap()
- hbStart := highbits(uint32(rangeStart))
- lbStart := lowbits(uint32(rangeStart))
- hbLast := highbits(uint32(rangeEnd - 1))
- lbLast := lowbits(uint32(rangeEnd - 1))
+ hbStart := uint32(highbits(uint32(rangeStart)))
+ lbStart := uint32(lowbits(uint32(rangeStart)))
+ hbLast := uint32(highbits(uint32(rangeEnd - 1)))
+ lbLast := uint32(lowbits(uint32(rangeEnd - 1)))
// copy the containers before the active area
- answer.highlowcontainer.appendCopiesUntil(bm.highlowcontainer, hbStart)
+ answer.highlowcontainer.appendCopiesUntil(bm.highlowcontainer, uint16(hbStart))
var max uint32 = maxLowBit
for hb := hbStart; hb <= hbLast; hb++ {
@@ -1262,23 +1396,23 @@ func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap {
containerLast = uint32(lbLast)
}
- i := bm.highlowcontainer.getIndex(hb)
- j := answer.highlowcontainer.getIndex(hb)
+ i := bm.highlowcontainer.getIndex(uint16(hb))
+ j := answer.highlowcontainer.getIndex(uint16(hb))
if i >= 0 {
c := bm.highlowcontainer.getContainerAtIndex(i).not(int(containerStart), int(containerLast)+1)
if c.getCardinality() > 0 {
- answer.highlowcontainer.insertNewKeyValueAt(-j-1, hb, c)
+ answer.highlowcontainer.insertNewKeyValueAt(-j-1, uint16(hb), c)
}
} else { // *think* the range of ones must never be
// empty.
- answer.highlowcontainer.insertNewKeyValueAt(-j-1, hb,
+ answer.highlowcontainer.insertNewKeyValueAt(-j-1, uint16(hb),
rangeOfOnes(int(containerStart), int(containerLast)))
}
}
// copy the containers after the active area.
- answer.highlowcontainer.appendCopiesAfter(bm.highlowcontainer, hbLast)
+ answer.highlowcontainer.appendCopiesAfter(bm.highlowcontainer, uint16(hbLast))
return answer
}
@@ -1296,6 +1430,21 @@ func (rb *Bitmap) GetCopyOnWrite() (val bool) {
return rb.highlowcontainer.copyOnWrite
}
+// CloneCopyOnWriteContainers clones all containers which have
+// needCopyOnWrite set to true.
+// This can be used to make sure it is safe to munmap a []byte
+// that the roaring array may still have a reference to, after
+// calling FromBuffer.
+// More generally this function is useful if you call FromBuffer
+// to construct a bitmap with a backing array buf
+// and then later discard the buf array. Note that you should call
+// CloneCopyOnWriteContainers on all bitmaps that were derived
+// from the 'FromBuffer' bitmap since they map have dependencies
+// on the buf array as well.
+func (rb *Bitmap) CloneCopyOnWriteContainers() {
+ rb.highlowcontainer.cloneCopyOnWriteContainers()
+}
+
// FlipInt calls Flip after casting the parameters (convenience method)
func FlipInt(bm *Bitmap, rangeStart, rangeEnd int) *Bitmap {
return Flip(bm, uint64(rangeStart), uint64(rangeEnd))
diff --git a/vendor/github.com/RoaringBitmap/roaring/roaringarray.go b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go
index d9659159d6..d9d5edda73 100644
--- a/vendor/github.com/RoaringBitmap/roaring/roaringarray.go
+++ b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go
@@ -4,16 +4,16 @@ import (
"bytes"
"encoding/binary"
"fmt"
- "io"
- "io/ioutil"
-
snappy "github.com/glycerine/go-unsnap-stream"
"github.com/tinylib/msgp/msgp"
+ "io"
)
//go:generate msgp -unexported
type container interface {
+ addOffset(uint16) []container
+
clone() container
and(container) container
andCardinality(container) int
@@ -37,7 +37,8 @@ type container interface {
not(start, final int) container // range is [firstOfRange,lastOfRange)
inot(firstOfRange, endx int) container // i stands for inplace, range is [firstOfRange,endx)
xor(r container) container
- getShortIterator() shortIterable
+ getShortIterator() shortPeekable
+ getReverseIterator() shortIterable
getManyIterator() manyIterable
contains(i uint16) bool
maximum() uint16
@@ -61,7 +62,6 @@ type container interface {
iremoveRange(start, final int) container // i stands for inplace, range is [firstOfRange,lastOfRange)
selectInt(x uint16) int // selectInt returns the xth integer in the container
serializedSizeInBytes() int
- readFrom(io.Reader) (int, error)
writeTo(io.Writer) (int, error)
numberOfRuns() int
@@ -280,6 +280,18 @@ func (ra *roaringArray) clone() *roaringArray {
return &sa
}
+// clone all containers which have needCopyOnWrite set to true
+// This can be used to make sure it is safe to munmap a []byte
+// that the roaring array may still have a reference to.
+func (ra *roaringArray) cloneCopyOnWriteContainers() {
+ for i, needCopyOnWrite := range ra.needCopyOnWrite {
+ if needCopyOnWrite {
+ ra.containers[i] = ra.containers[i].clone()
+ ra.needCopyOnWrite[i] = false
+ }
+ }
+}
+
// unused function:
//func (ra *roaringArray) containsKey(x uint16) bool {
// return (ra.binarySearch(0, int64(len(ra.keys)), x) >= 0)
@@ -456,8 +468,7 @@ func (ra *roaringArray) serializedSizeInBytes() uint64 {
//
// spec: https://github.com/RoaringBitmap/RoaringFormatSpec
//
-func (ra *roaringArray) toBytes() ([]byte, error) {
- stream := &bytes.Buffer{}
+func (ra *roaringArray) writeTo(w io.Writer) (n int64, err error) {
hasRun := ra.hasRunCompression()
isRunSizeInBytes := 0
cookieSize := 8
@@ -522,79 +533,77 @@ func (ra *roaringArray) toBytes() ([]byte, error) {
}
}
- _, err := stream.Write(buf[:nw])
+ written, err := w.Write(buf[:nw])
if err != nil {
- return nil, err
+ return n, err
}
- for i, c := range ra.containers {
- _ = i
- _, err := c.writeTo(stream)
+ n += int64(written)
+
+ for _, c := range ra.containers {
+ written, err := c.writeTo(w)
if err != nil {
- return nil, err
+ return n, err
}
+ n += int64(written)
}
- return stream.Bytes(), nil
+ return n, nil
}
//
// spec: https://github.com/RoaringBitmap/RoaringFormatSpec
//
-func (ra *roaringArray) writeTo(out io.Writer) (int64, error) {
- by, err := ra.toBytes()
- if err != nil {
- return 0, err
- }
- n, err := out.Write(by)
- if err == nil && n < len(by) {
- err = io.ErrShortWrite
- }
- return int64(n), err
+func (ra *roaringArray) toBytes() ([]byte, error) {
+ var buf bytes.Buffer
+ _, err := ra.writeTo(&buf)
+ return buf.Bytes(), err
}
-func (ra *roaringArray) fromBuffer(buf []byte) (int64, error) {
- pos := 0
- if len(buf) < 8 {
- return 0, fmt.Errorf("buffer too small, expecting at least 8 bytes, was %d", len(buf))
+func (ra *roaringArray) readFrom(stream byteInput) (int64, error) {
+ cookie, err := stream.readUInt32()
+
+ if err != nil {
+ return stream.getReadBytes(), fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err)
}
- cookie := binary.LittleEndian.Uint32(buf)
- pos += 4
- var size uint32 // number of containers
- haveRunContainers := false
+ var size uint32
var isRunBitmap []byte
- // cookie header
if cookie&0x0000FFFF == serialCookie {
- haveRunContainers = true
- size = uint32(uint16(cookie>>16) + 1) // number of containers
-
+ size = uint32(uint16(cookie>>16) + 1)
// create is-run-container bitmap
isRunBitmapSize := (int(size) + 7) / 8
- if pos+isRunBitmapSize > len(buf) {
- return 0, fmt.Errorf("malformed bitmap, is-run bitmap overruns buffer at %d", pos+isRunBitmapSize)
- }
+ isRunBitmap, err = stream.next(isRunBitmapSize)
- isRunBitmap = buf[pos : pos+isRunBitmapSize]
- pos += isRunBitmapSize
+ if err != nil {
+ return stream.getReadBytes(), fmt.Errorf("malformed bitmap, failed to read is-run bitmap, got: %s", err)
+ }
} else if cookie == serialCookieNoRunContainer {
- size = binary.LittleEndian.Uint32(buf[pos:])
- pos += 4
+ size, err = stream.readUInt32()
+
+ if err != nil {
+ return stream.getReadBytes(), fmt.Errorf("malformed bitmap, failed to read a bitmap size: %s", err)
+ }
} else {
- return 0, fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header")
+ return stream.getReadBytes(), fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header")
}
+
if size > (1 << 16) {
- return 0, fmt.Errorf("It is logically impossible to have more than (1<<16) containers.")
+ return stream.getReadBytes(), fmt.Errorf("it is logically impossible to have more than (1<<16) containers")
}
+
// descriptive header
- // keycard - is {key, cardinality} tuple slice
- if pos+2*2*int(size) > len(buf) {
- return 0, fmt.Errorf("malfomred bitmap, key-cardinality slice overruns buffer at %d", pos+2*2*int(size))
+ buf, err := stream.next(2 * 2 * int(size))
+
+ if err != nil {
+ return stream.getReadBytes(), fmt.Errorf("failed to read descriptive header: %s", err)
}
- keycard := byteSliceAsUint16Slice(buf[pos : pos+2*2*int(size)])
- pos += 2 * 2 * int(size)
- if !haveRunContainers || size >= noOffsetThreshold {
- pos += 4 * int(size)
+ keycard := byteSliceAsUint16Slice(buf)
+
+ if isRunBitmap == nil || size >= noOffsetThreshold {
+ if err := stream.skipBytes(int(size) * 4); err != nil {
+ return stream.getReadBytes(), fmt.Errorf("failed to skip bytes: %s", err)
+ }
}
// Allocate slices upfront as number of containers is known
@@ -603,11 +612,13 @@ func (ra *roaringArray) fromBuffer(buf []byte) (int64, error) {
} else {
ra.containers = make([]container, size)
}
+
if cap(ra.keys) >= int(size) {
ra.keys = ra.keys[:size]
} else {
ra.keys = make([]uint16, size)
}
+
if cap(ra.needCopyOnWrite) >= int(size) {
ra.needCopyOnWrite = ra.needCopyOnWrite[:size]
} else {
@@ -615,129 +626,62 @@ func (ra *roaringArray) fromBuffer(buf []byte) (int64, error) {
}
for i := uint32(0); i < size; i++ {
- key := uint16(keycard[2*i])
+ key := keycard[2*i]
card := int(keycard[2*i+1]) + 1
ra.keys[i] = key
ra.needCopyOnWrite[i] = true
- if haveRunContainers && isRunBitmap[i/8]&(1<<(i%8)) != 0 {
+ if isRunBitmap != nil && isRunBitmap[i/8]&(1<<(i%8)) != 0 {
// run container
- nr := binary.LittleEndian.Uint16(buf[pos:])
- pos += 2
- if pos+int(nr)*4 > len(buf) {
- return 0, fmt.Errorf("malformed bitmap, a run container overruns buffer at %d:%d", pos, pos+int(nr)*4)
+ nr, err := stream.readUInt16()
+
+ if err != nil {
+ return 0, fmt.Errorf("failed to read runtime container size: %s", err)
+ }
+
+ buf, err := stream.next(int(nr) * 4)
+
+ if err != nil {
+ return stream.getReadBytes(), fmt.Errorf("failed to read runtime container content: %s", err)
}
+
nb := runContainer16{
- iv: byteSliceAsInterval16Slice(buf[pos : pos+int(nr)*4]),
+ iv: byteSliceAsInterval16Slice(buf),
card: int64(card),
}
- pos += int(nr) * 4
+
ra.containers[i] = &nb
} else if card > arrayDefaultMaxSize {
// bitmap container
+ buf, err := stream.next(arrayDefaultMaxSize * 2)
+
+ if err != nil {
+ return stream.getReadBytes(), fmt.Errorf("failed to read bitmap container: %s", err)
+ }
+
nb := bitmapContainer{
cardinality: card,
- bitmap: byteSliceAsUint64Slice(buf[pos : pos+arrayDefaultMaxSize*2]),
+ bitmap: byteSliceAsUint64Slice(buf),
}
- pos += arrayDefaultMaxSize * 2
+
ra.containers[i] = &nb
} else {
// array container
- nb := arrayContainer{
- byteSliceAsUint16Slice(buf[pos : pos+card*2]),
- }
- pos += card * 2
- ra.containers[i] = &nb
- }
- }
-
- return int64(pos), nil
-}
+ buf, err := stream.next(card * 2)
-func (ra *roaringArray) readFrom(stream io.Reader) (int64, error) {
- pos := 0
- var cookie uint32
- err := binary.Read(stream, binary.LittleEndian, &cookie)
- if err != nil {
- return 0, fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err)
- }
- pos += 4
- var size uint32
- haveRunContainers := false
- var isRun *bitmapContainer
- if cookie&0x0000FFFF == serialCookie {
- haveRunContainers = true
- size = uint32(uint16(cookie>>16) + 1)
- bytesToRead := (int(size) + 7) / 8
- numwords := (bytesToRead + 7) / 8
- by := make([]byte, bytesToRead, numwords*8)
- nr, err := io.ReadFull(stream, by)
- if err != nil {
- return 8 + int64(nr), fmt.Errorf("error in readFrom: could not read the "+
- "runContainer bit flags of length %v bytes: %v", bytesToRead, err)
- }
- pos += bytesToRead
- by = by[:cap(by)]
- isRun = newBitmapContainer()
- for i := 0; i < numwords; i++ {
- isRun.bitmap[i] = binary.LittleEndian.Uint64(by)
- by = by[8:]
- }
- } else if cookie == serialCookieNoRunContainer {
- err = binary.Read(stream, binary.LittleEndian, &size)
- if err != nil {
- return 0, fmt.Errorf("error in roaringArray.readFrom: when reading size, got: %s", err)
- }
- pos += 4
- } else {
- return 0, fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header")
- }
- if size > (1 << 16) {
- return 0, fmt.Errorf("It is logically impossible to have more than (1<<16) containers.")
- }
- // descriptive header
- keycard := make([]uint16, 2*size, 2*size)
- err = binary.Read(stream, binary.LittleEndian, keycard)
- if err != nil {
- return 0, err
- }
- pos += 2 * 2 * int(size)
- // offset header
- if !haveRunContainers || size >= noOffsetThreshold {
- io.CopyN(ioutil.Discard, stream, 4*int64(size)) // we never skip ahead so this data can be ignored
- pos += 4 * int(size)
- }
- for i := uint32(0); i < size; i++ {
- key := int(keycard[2*i])
- card := int(keycard[2*i+1]) + 1
- if haveRunContainers && isRun.contains(uint16(i)) {
- nb := newRunContainer16()
- nr, err := nb.readFrom(stream)
if err != nil {
- return 0, err
+ return stream.getReadBytes(), fmt.Errorf("failed to read array container: %s", err)
}
- pos += nr
- ra.appendContainer(uint16(key), nb, false)
- } else if card > arrayDefaultMaxSize {
- nb := newBitmapContainer()
- nr, err := nb.readFrom(stream)
- if err != nil {
- return 0, err
- }
- nb.cardinality = card
- pos += nr
- ra.appendContainer(keycard[2*i], nb, false)
- } else {
- nb := newArrayContainerSize(card)
- nr, err := nb.readFrom(stream)
- if err != nil {
- return 0, err
+
+ nb := arrayContainer{
+ byteSliceAsUint16Slice(buf),
}
- pos += nr
- ra.appendContainer(keycard[2*i], nb, false)
+
+ ra.containers[i] = &nb
}
}
- return int64(pos), nil
+
+ return stream.getReadBytes(), nil
}
func (ra *roaringArray) hasRunCompression() bool {
diff --git a/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go b/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go
index 99fb0f6972..dcd718756a 100644
--- a/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go
+++ b/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go
@@ -8,7 +8,7 @@ import (
"github.com/tinylib/msgp/msgp"
)
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
func (z *containerSerz) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
@@ -48,7 +48,7 @@ func (z *containerSerz) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
func (z *containerSerz) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 2
// write "t"
@@ -72,7 +72,7 @@ func (z *containerSerz) EncodeMsg(en *msgp.Writer) (err error) {
return
}
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
func (z *containerSerz) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 2
@@ -88,7 +88,7 @@ func (z *containerSerz) MarshalMsg(b []byte) (o []byte, err error) {
return
}
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
func (z *containerSerz) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
@@ -129,13 +129,13 @@ func (z *containerSerz) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *containerSerz) Msgsize() (s int) {
s = 1 + 2 + msgp.Uint8Size + 2 + z.r.Msgsize()
return
}
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
func (z *contype) DecodeMsg(dc *msgp.Reader) (err error) {
{
var zajw uint8
@@ -148,7 +148,7 @@ func (z *contype) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
func (z contype) EncodeMsg(en *msgp.Writer) (err error) {
err = en.WriteUint8(uint8(z))
if err != nil {
@@ -157,14 +157,14 @@ func (z contype) EncodeMsg(en *msgp.Writer) (err error) {
return
}
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
func (z contype) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
o = msgp.AppendUint8(o, uint8(z))
return
}
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
func (z *contype) UnmarshalMsg(bts []byte) (o []byte, err error) {
{
var zwht uint8
@@ -178,13 +178,13 @@ func (z *contype) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z contype) Msgsize() (s int) {
s = msgp.Uint8Size
return
}
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
func (z *roaringArray) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
@@ -295,7 +295,7 @@ func (z *roaringArray) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
func (z *roaringArray) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 4
// write "keys"
@@ -370,7 +370,7 @@ func (z *roaringArray) EncodeMsg(en *msgp.Writer) (err error) {
return
}
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
func (z *roaringArray) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 4
@@ -407,7 +407,7 @@ func (z *roaringArray) MarshalMsg(b []byte) (o []byte, err error) {
return
}
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
func (z *roaringArray) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
@@ -519,7 +519,7 @@ func (z *roaringArray) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *roaringArray) Msgsize() (s int) {
s = 1 + 5 + msgp.ArrayHeaderSize + (len(z.keys) * (msgp.Uint16Size)) + 16 + msgp.ArrayHeaderSize + (len(z.needCopyOnWrite) * (msgp.BoolSize)) + 12 + msgp.BoolSize + 8 + msgp.ArrayHeaderSize
for zxhx := range z.conserz {
diff --git a/vendor/github.com/RoaringBitmap/roaring/rle16.go b/vendor/github.com/RoaringBitmap/roaring/runcontainer.go
index 951af65f3f..cbffdaf24d 100644
--- a/vendor/github.com/RoaringBitmap/roaring/rle16.go
+++ b/vendor/github.com/RoaringBitmap/roaring/runcontainer.go
@@ -853,6 +853,21 @@ func (rc *runContainer16) numIntervals() int {
return len(rc.iv)
}
+// searchOptions allows us to accelerate search with
+// prior knowledge of (mostly lower) bounds. This is used by Union
+// and Intersect.
+type searchOptions struct {
+ // start here instead of at 0
+ startIndex int64
+
+ // upper bound instead of len(rc.iv);
+ // endxIndex == 0 means ignore the bound and use
+ // endxIndex == n ==len(rc.iv) which is also
+ // naturally the default for search()
+ // when opt = nil.
+ endxIndex int64
+}
+
// search returns alreadyPresent to indicate if the
// key is already in one of our interval16s.
//
@@ -1134,135 +1149,163 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) {
//msgp:ignore runIterator
-// runIterator16 advice: you must call Next() at least once
-// before calling Cur(); and you should call HasNext()
-// before calling Next() to insure there are contents.
+// runIterator16 advice: you must call hasNext()
+// before calling next()/peekNext() to insure there are contents.
type runIterator16 struct {
rc *runContainer16
curIndex int64
curPosInIndex uint16
- curSeq int64
}
// newRunIterator16 returns a new empty run container.
func (rc *runContainer16) newRunIterator16() *runIterator16 {
- return &runIterator16{rc: rc, curIndex: -1}
+ return &runIterator16{rc: rc, curIndex: 0, curPosInIndex: 0}
}
-// HasNext returns false if calling Next will panic. It
+// hasNext returns false if calling next will panic. It
// returns true when there is at least one more value
// available in the iteration sequence.
func (ri *runIterator16) hasNext() bool {
- if len(ri.rc.iv) == 0 {
- return false
- }
- if ri.curIndex == -1 {
- return true
+ return int64(len(ri.rc.iv)) > ri.curIndex+1 ||
+ (int64(len(ri.rc.iv)) == ri.curIndex+1 && ri.rc.iv[ri.curIndex].length >= ri.curPosInIndex)
+}
+
+// next returns the next value in the iteration sequence.
+func (ri *runIterator16) next() uint16 {
+ next := ri.rc.iv[ri.curIndex].start + ri.curPosInIndex
+
+ if ri.curPosInIndex == ri.rc.iv[ri.curIndex].length {
+ ri.curPosInIndex = 0
+ ri.curIndex++
+ } else {
+ ri.curPosInIndex++
}
- return ri.curSeq+1 < ri.rc.cardinality()
+
+ return next
}
-// cur returns the current value pointed to by the iterator.
-func (ri *runIterator16) cur() uint16 {
+// peekNext returns the next value in the iteration sequence without advancing the iterator
+func (ri *runIterator16) peekNext() uint16 {
return ri.rc.iv[ri.curIndex].start + ri.curPosInIndex
}
-// Next returns the next value in the iteration sequence.
-func (ri *runIterator16) next() uint16 {
- if !ri.hasNext() {
- panic("no Next available")
+// advanceIfNeeded advances as long as the next value is smaller than minval
+func (ri *runIterator16) advanceIfNeeded(minval uint16) {
+ if !ri.hasNext() || ri.peekNext() >= minval {
+ return
}
- if ri.curIndex >= int64(len(ri.rc.iv)) {
- panic("runIterator.Next() going beyond what is available")
+
+ opt := &searchOptions{
+ startIndex: ri.curIndex,
+ endxIndex: int64(len(ri.rc.iv)),
}
- if ri.curIndex == -1 {
- // first time is special
- ri.curIndex = 0
+
+ // interval cannot be -1 because of minval > peekNext
+ interval, isPresent, _ := ri.rc.search(int64(minval), opt)
+
+ // if the minval is present, set the curPosIndex at the right position
+ if isPresent {
+ ri.curIndex = interval
+ ri.curPosInIndex = minval - ri.rc.iv[ri.curIndex].start
} else {
- ri.curPosInIndex++
- if int64(ri.rc.iv[ri.curIndex].start)+int64(ri.curPosInIndex) == int64(ri.rc.iv[ri.curIndex].last())+1 {
- ri.curPosInIndex = 0
- ri.curIndex++
- }
- ri.curSeq++
+ // otherwise interval is set to to the minimum index of rc.iv
+ // which comes strictly before the key, that's why we set the next interval
+ ri.curIndex = interval + 1
+ ri.curPosInIndex = 0
}
- return ri.cur()
}
-// remove removes the element that the iterator
-// is on from the run container. You can use
-// Cur if you want to double check what is about
-// to be deleted.
-func (ri *runIterator16) remove() uint16 {
- n := ri.rc.cardinality()
- if n == 0 {
- panic("runIterator.Remove called on empty runContainer16")
+// runReverseIterator16 advice: you must call hasNext()
+// before calling next() to insure there are contents.
+type runReverseIterator16 struct {
+ rc *runContainer16
+ curIndex int64 // index into rc.iv
+ curPosInIndex uint16 // offset in rc.iv[curIndex]
+}
+
+// newRunReverseIterator16 returns a new empty run iterator.
+func (rc *runContainer16) newRunReverseIterator16() *runReverseIterator16 {
+ index := int64(len(rc.iv)) - 1
+ pos := uint16(0)
+
+ if index >= 0 {
+ pos = rc.iv[index].length
}
- cur := ri.cur()
- ri.rc.deleteAt(&ri.curIndex, &ri.curPosInIndex, &ri.curSeq)
- return cur
+ return &runReverseIterator16{
+ rc: rc,
+ curIndex: index,
+ curPosInIndex: pos,
+ }
}
-type manyRunIterator16 struct {
- rc *runContainer16
- curIndex int64
- curPosInIndex uint16
- curSeq int64
+// hasNext returns false if calling next will panic. It
+// returns true when there is at least one more value
+// available in the iteration sequence.
+func (ri *runReverseIterator16) hasNext() bool {
+ return ri.curIndex > 0 || ri.curIndex == 0 && ri.curPosInIndex >= 0
}
-func (rc *runContainer16) newManyRunIterator16() *manyRunIterator16 {
- return &manyRunIterator16{rc: rc, curIndex: -1}
-}
+// next returns the next value in the iteration sequence.
+func (ri *runReverseIterator16) next() uint16 {
+ next := ri.rc.iv[ri.curIndex].start + ri.curPosInIndex
-func (ri *manyRunIterator16) hasNext() bool {
- if len(ri.rc.iv) == 0 {
- return false
- }
- if ri.curIndex == -1 {
- return true
+ if ri.curPosInIndex > 0 {
+ ri.curPosInIndex--
+ } else {
+ ri.curIndex--
+
+ if ri.curIndex >= 0 {
+ ri.curPosInIndex = ri.rc.iv[ri.curIndex].length
+ }
}
- return ri.curSeq+1 < ri.rc.cardinality()
+
+ return next
+}
+
+func (rc *runContainer16) newManyRunIterator16() *runIterator16 {
+ return rc.newRunIterator16()
}
// hs are the high bits to include to avoid needing to reiterate over the buffer in NextMany
-func (ri *manyRunIterator16) nextMany(hs uint32, buf []uint32) int {
+func (ri *runIterator16) nextMany(hs uint32, buf []uint32) int {
n := 0
+
if !ri.hasNext() {
return n
}
+
// start and end are inclusive
for n < len(buf) {
- if ri.curIndex == -1 || int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex) <= 0 {
+ moreVals := 0
+
+ if ri.rc.iv[ri.curIndex].length >= ri.curPosInIndex {
+ // add as many as you can from this seq
+ moreVals = minOfInt(int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex)+1, len(buf)-n)
+ base := uint32(ri.rc.iv[ri.curIndex].start+ri.curPosInIndex) | hs
+
+ // allows BCE
+ buf2 := buf[n : n+moreVals]
+ for i := range buf2 {
+ buf2[i] = base + uint32(i)
+ }
+
+ // update values
+ n += moreVals
+ }
+
+ if moreVals+int(ri.curPosInIndex) > int(ri.rc.iv[ri.curIndex].length) {
ri.curPosInIndex = 0
ri.curIndex++
+
if ri.curIndex == int64(len(ri.rc.iv)) {
break
}
- buf[n] = uint32(ri.rc.iv[ri.curIndex].start) | hs
- if ri.curIndex != 0 {
- ri.curSeq += 1
- }
- n += 1
- // not strictly necessarily due to len(buf)-n min check, but saves some work
- continue
- }
- // add as many as you can from this seq
- moreVals := minOfInt(int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex), len(buf)-n)
-
- base := uint32(ri.rc.iv[ri.curIndex].start+ri.curPosInIndex+1) | hs
-
- // allows BCE
- buf2 := buf[n : n+moreVals]
- for i := range buf2 {
- buf2[i] = base + uint32(i)
+ } else {
+ ri.curPosInIndex += uint16(moreVals) //moreVals always fits in uint16
}
-
- // update values
- ri.curPosInIndex += uint16(moreVals) //moreVals always fits in uint16
- ri.curSeq += int64(moreVals)
- n += moreVals
}
+
return n
}
@@ -1270,21 +1313,19 @@ func (ri *manyRunIterator16) nextMany(hs uint32, buf []uint32) int {
func (rc *runContainer16) removeKey(key uint16) (wasPresent bool) {
var index int64
- var curSeq int64
index, wasPresent, _ = rc.search(int64(key), nil)
if !wasPresent {
return // already removed, nothing to do.
}
pos := key - rc.iv[index].start
- rc.deleteAt(&index, &pos, &curSeq)
+ rc.deleteAt(&index, &pos)
return
}
// internal helper functions
-func (rc *runContainer16) deleteAt(curIndex *int64, curPosInIndex *uint16, curSeq *int64) {
+func (rc *runContainer16) deleteAt(curIndex *int64, curPosInIndex *uint16) {
rc.card--
- *curSeq--
ci := *curIndex
pos := *curPosInIndex
@@ -1401,7 +1442,7 @@ func (rc *runContainer16) selectInt16(j uint16) int {
var offset int64
for k := range rc.iv {
- nextOffset := offset + rc.iv[k].runlen() + 1
+ nextOffset := offset + rc.iv[k].runlen()
if nextOffset > int64(j) {
return int(int64(rc.iv[k].start) + (int64(j) - offset))
}
@@ -1724,24 +1765,750 @@ func (rc *runContainer16) containerType() contype {
}
func (rc *runContainer16) equals16(srb *runContainer16) bool {
- //p("both rc16")
// Check if the containers are the same object.
if rc == srb {
- //p("same object")
return true
}
if len(srb.iv) != len(rc.iv) {
- //p("iv len differ")
return false
}
for i, v := range rc.iv {
if v != srb.iv[i] {
- //p("differ at iv i=%v, srb.iv[i]=%v, rc.iv[i]=%v", i, srb.iv[i], rc.iv[i])
return false
}
}
- //p("all intervals same, returning true")
return true
}
+
+// compile time verify we meet interface requirements
+var _ container = &runContainer16{}
+
+func (rc *runContainer16) clone() container {
+ return newRunContainer16CopyIv(rc.iv)
+}
+
+func (rc *runContainer16) minimum() uint16 {
+ return rc.iv[0].start // assume not empty
+}
+
+func (rc *runContainer16) maximum() uint16 {
+ return rc.iv[len(rc.iv)-1].last() // assume not empty
+}
+
+func (rc *runContainer16) isFull() bool {
+ return (len(rc.iv) == 1) && ((rc.iv[0].start == 0) && (rc.iv[0].last() == MaxUint16))
+}
+
+func (rc *runContainer16) and(a container) container {
+ if rc.isFull() {
+ return a.clone()
+ }
+ switch c := a.(type) {
+ case *runContainer16:
+ return rc.intersect(c)
+ case *arrayContainer:
+ return rc.andArray(c)
+ case *bitmapContainer:
+ return rc.andBitmapContainer(c)
+ }
+ panic("unsupported container type")
+}
+
+func (rc *runContainer16) andCardinality(a container) int {
+ switch c := a.(type) {
+ case *runContainer16:
+ return int(rc.intersectCardinality(c))
+ case *arrayContainer:
+ return rc.andArrayCardinality(c)
+ case *bitmapContainer:
+ return rc.andBitmapContainerCardinality(c)
+ }
+ panic("unsupported container type")
+}
+
+// andBitmapContainer finds the intersection of rc and b.
+func (rc *runContainer16) andBitmapContainer(bc *bitmapContainer) container {
+ bc2 := newBitmapContainerFromRun(rc)
+ return bc2.andBitmap(bc)
+}
+
+func (rc *runContainer16) andArrayCardinality(ac *arrayContainer) int {
+ pos := 0
+ answer := 0
+ maxpos := ac.getCardinality()
+ if maxpos == 0 {
+ return 0 // won't happen in actual code
+ }
+ v := ac.content[pos]
+mainloop:
+ for _, p := range rc.iv {
+ for v < p.start {
+ pos++
+ if pos == maxpos {
+ break mainloop
+ }
+ v = ac.content[pos]
+ }
+ for v <= p.last() {
+ answer++
+ pos++
+ if pos == maxpos {
+ break mainloop
+ }
+ v = ac.content[pos]
+ }
+ }
+ return answer
+}
+
+func (rc *runContainer16) iand(a container) container {
+ if rc.isFull() {
+ return a.clone()
+ }
+ switch c := a.(type) {
+ case *runContainer16:
+ return rc.inplaceIntersect(c)
+ case *arrayContainer:
+ return rc.andArray(c)
+ case *bitmapContainer:
+ return rc.iandBitmapContainer(c)
+ }
+ panic("unsupported container type")
+}
+
+func (rc *runContainer16) inplaceIntersect(rc2 *runContainer16) container {
+ // TODO: optimize by doing less allocation, possibly?
+ // sect will be new
+ sect := rc.intersect(rc2)
+ *rc = *sect
+ return rc
+}
+
+func (rc *runContainer16) iandBitmapContainer(bc *bitmapContainer) container {
+ isect := rc.andBitmapContainer(bc)
+ *rc = *newRunContainer16FromContainer(isect)
+ return rc
+}
+
+func (rc *runContainer16) andArray(ac *arrayContainer) container {
+ if len(rc.iv) == 0 {
+ return newArrayContainer()
+ }
+
+ acCardinality := ac.getCardinality()
+ c := newArrayContainerCapacity(acCardinality)
+
+ for rlePos, arrayPos := 0, 0; arrayPos < acCardinality; {
+ iv := rc.iv[rlePos]
+ arrayVal := ac.content[arrayPos]
+
+ for iv.last() < arrayVal {
+ rlePos++
+ if rlePos == len(rc.iv) {
+ return c
+ }
+ iv = rc.iv[rlePos]
+ }
+
+ if iv.start > arrayVal {
+ arrayPos = advanceUntil(ac.content, arrayPos, len(ac.content), iv.start)
+ } else {
+ c.content = append(c.content, arrayVal)
+ arrayPos++
+ }
+ }
+ return c
+}
+
+func (rc *runContainer16) andNot(a container) container {
+ switch c := a.(type) {
+ case *arrayContainer:
+ return rc.andNotArray(c)
+ case *bitmapContainer:
+ return rc.andNotBitmap(c)
+ case *runContainer16:
+ return rc.andNotRunContainer16(c)
+ }
+ panic("unsupported container type")
+}
+
+func (rc *runContainer16) fillLeastSignificant16bits(x []uint32, i int, mask uint32) {
+ k := 0
+ var val int64
+ for _, p := range rc.iv {
+ n := p.runlen()
+ for j := int64(0); j < n; j++ {
+ val = int64(p.start) + j
+ x[k+i] = uint32(val) | mask
+ k++
+ }
+ }
+}
+
+func (rc *runContainer16) getShortIterator() shortPeekable {
+ return rc.newRunIterator16()
+}
+
+func (rc *runContainer16) getReverseIterator() shortIterable {
+ return rc.newRunReverseIterator16()
+}
+
+func (rc *runContainer16) getManyIterator() manyIterable {
+ return rc.newManyRunIterator16()
+}
+
+// add the values in the range [firstOfRange, endx). endx
+// is still abe to express 2^16 because it is an int not an uint16.
+func (rc *runContainer16) iaddRange(firstOfRange, endx int) container {
+
+ if firstOfRange >= endx {
+ panic(fmt.Sprintf("invalid %v = endx >= firstOfRange", endx))
+ }
+ addme := newRunContainer16TakeOwnership([]interval16{
+ {
+ start: uint16(firstOfRange),
+ length: uint16(endx - 1 - firstOfRange),
+ },
+ })
+ *rc = *rc.union(addme)
+ return rc
+}
+
+// remove the values in the range [firstOfRange,endx)
+func (rc *runContainer16) iremoveRange(firstOfRange, endx int) container {
+ if firstOfRange >= endx {
+ panic(fmt.Sprintf("request to iremove empty set [%v, %v),"+
+ " nothing to do.", firstOfRange, endx))
+ //return rc
+ }
+ x := newInterval16Range(uint16(firstOfRange), uint16(endx-1))
+ rc.isubtract(x)
+ return rc
+}
+
+// not flip the values in the range [firstOfRange,endx)
+func (rc *runContainer16) not(firstOfRange, endx int) container {
+ if firstOfRange >= endx {
+ panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange))
+ }
+
+ return rc.Not(firstOfRange, endx)
+}
+
+// Not flips the values in the range [firstOfRange,endx).
+// This is not inplace. Only the returned value has the flipped bits.
+//
+// Currently implemented as (!A intersect B) union (A minus B),
+// where A is rc, and B is the supplied [firstOfRange, endx) interval.
+//
+// TODO(time optimization): convert this to a single pass
+// algorithm by copying AndNotRunContainer16() and modifying it.
+// Current routine is correct but
+// makes 2 more passes through the arrays than should be
+// strictly necessary. Measure both ways though--this may not matter.
+//
+func (rc *runContainer16) Not(firstOfRange, endx int) *runContainer16 {
+
+ if firstOfRange >= endx {
+ panic(fmt.Sprintf("invalid %v = endx >= firstOfRange == %v", endx, firstOfRange))
+ }
+
+ if firstOfRange >= endx {
+ return rc.Clone()
+ }
+
+ a := rc
+ // algo:
+ // (!A intersect B) union (A minus B)
+
+ nota := a.invert()
+
+ bs := []interval16{newInterval16Range(uint16(firstOfRange), uint16(endx-1))}
+ b := newRunContainer16TakeOwnership(bs)
+
+ notAintersectB := nota.intersect(b)
+
+ aMinusB := a.AndNotRunContainer16(b)
+
+ rc2 := notAintersectB.union(aMinusB)
+ return rc2
+}
+
+// equals is now logical equals; it does not require the
+// same underlying container type.
+func (rc *runContainer16) equals(o container) bool {
+ srb, ok := o.(*runContainer16)
+
+ if !ok {
+ // maybe value instead of pointer
+ val, valok := o.(*runContainer16)
+ if valok {
+ srb = val
+ ok = true
+ }
+ }
+ if ok {
+ // Check if the containers are the same object.
+ if rc == srb {
+ return true
+ }
+
+ if len(srb.iv) != len(rc.iv) {
+ return false
+ }
+
+ for i, v := range rc.iv {
+ if v != srb.iv[i] {
+ return false
+ }
+ }
+ return true
+ }
+
+ // use generic comparison
+ if o.getCardinality() != rc.getCardinality() {
+ return false
+ }
+ rit := rc.getShortIterator()
+ bit := o.getShortIterator()
+
+ //k := 0
+ for rit.hasNext() {
+ if bit.next() != rit.next() {
+ return false
+ }
+ //k++
+ }
+ return true
+}
+
+func (rc *runContainer16) iaddReturnMinimized(x uint16) container {
+ rc.Add(x)
+ return rc
+}
+
+func (rc *runContainer16) iadd(x uint16) (wasNew bool) {
+ return rc.Add(x)
+}
+
+func (rc *runContainer16) iremoveReturnMinimized(x uint16) container {
+ rc.removeKey(x)
+ return rc
+}
+
+func (rc *runContainer16) iremove(x uint16) bool {
+ return rc.removeKey(x)
+}
+
+func (rc *runContainer16) or(a container) container {
+ if rc.isFull() {
+ return rc.clone()
+ }
+ switch c := a.(type) {
+ case *runContainer16:
+ return rc.union(c)
+ case *arrayContainer:
+ return rc.orArray(c)
+ case *bitmapContainer:
+ return rc.orBitmapContainer(c)
+ }
+ panic("unsupported container type")
+}
+
+func (rc *runContainer16) orCardinality(a container) int {
+ switch c := a.(type) {
+ case *runContainer16:
+ return int(rc.unionCardinality(c))
+ case *arrayContainer:
+ return rc.orArrayCardinality(c)
+ case *bitmapContainer:
+ return rc.orBitmapContainerCardinality(c)
+ }
+ panic("unsupported container type")
+}
+
+// orBitmapContainer finds the union of rc and bc.
+func (rc *runContainer16) orBitmapContainer(bc *bitmapContainer) container {
+ bc2 := newBitmapContainerFromRun(rc)
+ return bc2.iorBitmap(bc)
+}
+
+func (rc *runContainer16) andBitmapContainerCardinality(bc *bitmapContainer) int {
+ answer := 0
+ for i := range rc.iv {
+ answer += bc.getCardinalityInRange(uint(rc.iv[i].start), uint(rc.iv[i].last())+1)
+ }
+ //bc.computeCardinality()
+ return answer
+}
+
+func (rc *runContainer16) orBitmapContainerCardinality(bc *bitmapContainer) int {
+ return rc.getCardinality() + bc.getCardinality() - rc.andBitmapContainerCardinality(bc)
+}
+
+// orArray finds the union of rc and ac.
+func (rc *runContainer16) orArray(ac *arrayContainer) container {
+ bc1 := newBitmapContainerFromRun(rc)
+ bc2 := ac.toBitmapContainer()
+ return bc1.orBitmap(bc2)
+}
+
+// orArray finds the union of rc and ac.
+func (rc *runContainer16) orArrayCardinality(ac *arrayContainer) int {
+ return ac.getCardinality() + rc.getCardinality() - rc.andArrayCardinality(ac)
+}
+
+func (rc *runContainer16) ior(a container) container {
+ if rc.isFull() {
+ return rc
+ }
+ switch c := a.(type) {
+ case *runContainer16:
+ return rc.inplaceUnion(c)
+ case *arrayContainer:
+ return rc.iorArray(c)
+ case *bitmapContainer:
+ return rc.iorBitmapContainer(c)
+ }
+ panic("unsupported container type")
+}
+
+func (rc *runContainer16) inplaceUnion(rc2 *runContainer16) container {
+ for _, p := range rc2.iv {
+ last := int64(p.last())
+ for i := int64(p.start); i <= last; i++ {
+ rc.Add(uint16(i))
+ }
+ }
+ return rc
+}
+
+func (rc *runContainer16) iorBitmapContainer(bc *bitmapContainer) container {
+
+ it := bc.getShortIterator()
+ for it.hasNext() {
+ rc.Add(it.next())
+ }
+ return rc
+}
+
+func (rc *runContainer16) iorArray(ac *arrayContainer) container {
+ it := ac.getShortIterator()
+ for it.hasNext() {
+ rc.Add(it.next())
+ }
+ return rc
+}
+
+// lazyIOR is described (not yet implemented) in
+// this nice note from @lemire on
+// https://github.com/RoaringBitmap/roaring/pull/70#issuecomment-263613737
+//
+// Description of lazyOR and lazyIOR from @lemire:
+//
+// Lazy functions are optional and can be simply
+// wrapper around non-lazy functions.
+//
+// The idea of "laziness" is as follows. It is
+// inspired by the concept of lazy evaluation
+// you might be familiar with (functional programming
+// and all that). So a roaring bitmap is
+// such that all its containers are, in some
+// sense, chosen to use as little memory as
+// possible. This is nice. Also, all bitsets
+// are "cardinality aware" so that you can do
+// fast rank/select queries, or query the
+// cardinality of the whole bitmap... very fast,
+// without latency.
+//
+// However, imagine that you are aggregating 100
+// bitmaps together. So you OR the first two, then OR
+// that with the third one and so forth. Clearly,
+// intermediate bitmaps don't need to be as
+// compressed as possible, right? They can be
+// in a "dirty state". You only need the end
+// result to be in a nice state... which you
+// can achieve by calling repairAfterLazy at the end.
+//
+// The Java/C code does something special for
+// the in-place lazy OR runs. The idea is that
+// instead of taking two run containers and
+// generating a new one, we actually try to
+// do the computation in-place through a
+// technique invented by @gssiyankai (pinging him!).
+// What you do is you check whether the host
+// run container has lots of extra capacity.
+// If it does, you move its data at the end of
+// the backing array, and then you write
+// the answer at the beginning. What this
+// trick does is minimize memory allocations.
+//
+func (rc *runContainer16) lazyIOR(a container) container {
+ // not lazy at the moment
+ return rc.ior(a)
+}
+
+// lazyOR is described above in lazyIOR.
+func (rc *runContainer16) lazyOR(a container) container {
+ // not lazy at the moment
+ return rc.or(a)
+}
+
+func (rc *runContainer16) intersects(a container) bool {
+ // TODO: optimize by doing inplace/less allocation, possibly?
+ isect := rc.and(a)
+ return isect.getCardinality() > 0
+}
+
+func (rc *runContainer16) xor(a container) container {
+ switch c := a.(type) {
+ case *arrayContainer:
+ return rc.xorArray(c)
+ case *bitmapContainer:
+ return rc.xorBitmap(c)
+ case *runContainer16:
+ return rc.xorRunContainer16(c)
+ }
+ panic("unsupported container type")
+}
+
+func (rc *runContainer16) iandNot(a container) container {
+ switch c := a.(type) {
+ case *arrayContainer:
+ return rc.iandNotArray(c)
+ case *bitmapContainer:
+ return rc.iandNotBitmap(c)
+ case *runContainer16:
+ return rc.iandNotRunContainer16(c)
+ }
+ panic("unsupported container type")
+}
+
+// flip the values in the range [firstOfRange,endx)
+func (rc *runContainer16) inot(firstOfRange, endx int) container {
+ if firstOfRange >= endx {
+ panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange))
+ }
+ // TODO: minimize copies, do it all inplace; not() makes a copy.
+ rc = rc.Not(firstOfRange, endx)
+ return rc
+}
+
+func (rc *runContainer16) getCardinality() int {
+ return int(rc.cardinality())
+}
+
+func (rc *runContainer16) rank(x uint16) int {
+ n := int64(len(rc.iv))
+ xx := int64(x)
+ w, already, _ := rc.search(xx, nil)
+ if w < 0 {
+ return 0
+ }
+ if !already && w == n-1 {
+ return rc.getCardinality()
+ }
+ var rnk int64
+ if !already {
+ for i := int64(0); i <= w; i++ {
+ rnk += rc.iv[i].runlen()
+ }
+ return int(rnk)
+ }
+ for i := int64(0); i < w; i++ {
+ rnk += rc.iv[i].runlen()
+ }
+ rnk += int64(x-rc.iv[w].start) + 1
+ return int(rnk)
+}
+
+func (rc *runContainer16) selectInt(x uint16) int {
+ return rc.selectInt16(x)
+}
+
+func (rc *runContainer16) andNotRunContainer16(b *runContainer16) container {
+ return rc.AndNotRunContainer16(b)
+}
+
+func (rc *runContainer16) andNotArray(ac *arrayContainer) container {
+ rcb := rc.toBitmapContainer()
+ acb := ac.toBitmapContainer()
+ return rcb.andNotBitmap(acb)
+}
+
+func (rc *runContainer16) andNotBitmap(bc *bitmapContainer) container {
+ rcb := rc.toBitmapContainer()
+ return rcb.andNotBitmap(bc)
+}
+
+func (rc *runContainer16) toBitmapContainer() *bitmapContainer {
+ bc := newBitmapContainer()
+ for i := range rc.iv {
+ bc.iaddRange(int(rc.iv[i].start), int(rc.iv[i].last())+1)
+ }
+ bc.computeCardinality()
+ return bc
+}
+
+func (rc *runContainer16) iandNotRunContainer16(x2 *runContainer16) container {
+ rcb := rc.toBitmapContainer()
+ x2b := x2.toBitmapContainer()
+ rcb.iandNotBitmapSurely(x2b)
+ // TODO: check size and optimize the return value
+ // TODO: is inplace modification really required? If not, elide the copy.
+ rc2 := newRunContainer16FromBitmapContainer(rcb)
+ *rc = *rc2
+ return rc
+}
+
+func (rc *runContainer16) iandNotArray(ac *arrayContainer) container {
+ rcb := rc.toBitmapContainer()
+ acb := ac.toBitmapContainer()
+ rcb.iandNotBitmapSurely(acb)
+ // TODO: check size and optimize the return value
+ // TODO: is inplace modification really required? If not, elide the copy.
+ rc2 := newRunContainer16FromBitmapContainer(rcb)
+ *rc = *rc2
+ return rc
+}
+
+func (rc *runContainer16) iandNotBitmap(bc *bitmapContainer) container {
+ rcb := rc.toBitmapContainer()
+ rcb.iandNotBitmapSurely(bc)
+ // TODO: check size and optimize the return value
+ // TODO: is inplace modification really required? If not, elide the copy.
+ rc2 := newRunContainer16FromBitmapContainer(rcb)
+ *rc = *rc2
+ return rc
+}
+
+func (rc *runContainer16) xorRunContainer16(x2 *runContainer16) container {
+ rcb := rc.toBitmapContainer()
+ x2b := x2.toBitmapContainer()
+ return rcb.xorBitmap(x2b)
+}
+
+func (rc *runContainer16) xorArray(ac *arrayContainer) container {
+ rcb := rc.toBitmapContainer()
+ acb := ac.toBitmapContainer()
+ return rcb.xorBitmap(acb)
+}
+
+func (rc *runContainer16) xorBitmap(bc *bitmapContainer) container {
+ rcb := rc.toBitmapContainer()
+ return rcb.xorBitmap(bc)
+}
+
+// convert to bitmap or array *if needed*
+func (rc *runContainer16) toEfficientContainer() container {
+
+ // runContainer16SerializedSizeInBytes(numRuns)
+ sizeAsRunContainer := rc.getSizeInBytes()
+ sizeAsBitmapContainer := bitmapContainerSizeInBytes()
+ card := int(rc.cardinality())
+ sizeAsArrayContainer := arrayContainerSizeInBytes(card)
+ if sizeAsRunContainer <= minOfInt(sizeAsBitmapContainer, sizeAsArrayContainer) {
+ return rc
+ }
+ if card <= arrayDefaultMaxSize {
+ return rc.toArrayContainer()
+ }
+ bc := newBitmapContainerFromRun(rc)
+ return bc
+}
+
+func (rc *runContainer16) toArrayContainer() *arrayContainer {
+ ac := newArrayContainer()
+ for i := range rc.iv {
+ ac.iaddRange(int(rc.iv[i].start), int(rc.iv[i].last())+1)
+ }
+ return ac
+}
+
+func newRunContainer16FromContainer(c container) *runContainer16 {
+
+ switch x := c.(type) {
+ case *runContainer16:
+ return x.Clone()
+ case *arrayContainer:
+ return newRunContainer16FromArray(x)
+ case *bitmapContainer:
+ return newRunContainer16FromBitmapContainer(x)
+ }
+ panic("unsupported container type")
+}
+
+// And finds the intersection of rc and b.
+func (rc *runContainer16) And(b *Bitmap) *Bitmap {
+ out := NewBitmap()
+ for _, p := range rc.iv {
+ plast := p.last()
+ for i := p.start; i <= plast; i++ {
+ if b.Contains(uint32(i)) {
+ out.Add(uint32(i))
+ }
+ }
+ }
+ return out
+}
+
+// Xor returns the exclusive-or of rc and b.
+func (rc *runContainer16) Xor(b *Bitmap) *Bitmap {
+ out := b.Clone()
+ for _, p := range rc.iv {
+ plast := p.last()
+ for v := p.start; v <= plast; v++ {
+ w := uint32(v)
+ if out.Contains(w) {
+ out.RemoveRange(uint64(w), uint64(w+1))
+ } else {
+ out.Add(w)
+ }
+ }
+ }
+ return out
+}
+
+// Or returns the union of rc and b.
+func (rc *runContainer16) Or(b *Bitmap) *Bitmap {
+ out := b.Clone()
+ for _, p := range rc.iv {
+ plast := p.last()
+ for v := p.start; v <= plast; v++ {
+ out.Add(uint32(v))
+ }
+ }
+ return out
+}
+
+// serializedSizeInBytes returns the number of bytes of memory
+// required by this runContainer16. This is for the
+// Roaring format, as specified https://github.com/RoaringBitmap/RoaringFormatSpec/
+func (rc *runContainer16) serializedSizeInBytes() int {
+ // number of runs in one uint16, then each run
+ // needs two more uint16
+ return 2 + len(rc.iv)*4
+}
+
+func (rc *runContainer16) addOffset(x uint16) []container {
+ low := newRunContainer16()
+ high := newRunContainer16()
+
+ for _, iv := range rc.iv {
+ val := int(iv.start) + int(x)
+ finalVal := int(val) + int(iv.length)
+ if val <= 0xffff {
+ if finalVal <= 0xffff {
+ low.iv = append(low.iv, interval16{uint16(val), iv.length})
+ } else {
+ low.iv = append(low.iv, interval16{uint16(val), uint16(0xffff - val)})
+ high.iv = append(high.iv, interval16{uint16(0), uint16(finalVal & 0xffff)})
+ }
+ } else {
+ high.iv = append(high.iv, interval16{uint16(val & 0xffff), iv.length})
+ }
+ }
+ return []container{low, high}
+}
diff --git a/vendor/github.com/RoaringBitmap/roaring/rle16_gen.go b/vendor/github.com/RoaringBitmap/roaring/runcontainer_gen.go
index 05bf4463f1..84537d087f 100644
--- a/vendor/github.com/RoaringBitmap/roaring/rle16_gen.go
+++ b/vendor/github.com/RoaringBitmap/roaring/runcontainer_gen.go
@@ -6,7 +6,7 @@ package roaring
import "github.com/tinylib/msgp/msgp"
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
func (z *addHelper16) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
@@ -169,7 +169,7 @@ func (z *addHelper16) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
func (z *addHelper16) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 5
// write "runstart"
@@ -284,7 +284,7 @@ func (z *addHelper16) EncodeMsg(en *msgp.Writer) (err error) {
return
}
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
func (z *addHelper16) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 5
@@ -334,7 +334,7 @@ func (z *addHelper16) MarshalMsg(b []byte) (o []byte, err error) {
return
}
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
func (z *addHelper16) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
@@ -498,7 +498,7 @@ func (z *addHelper16) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *addHelper16) Msgsize() (s int) {
s = 1 + 9 + msgp.Uint16Size + 7 + msgp.Uint16Size + 14 + msgp.Uint16Size + 2 + msgp.ArrayHeaderSize + (len(z.m) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 3
if z.rc == nil {
@@ -509,7 +509,7 @@ func (z *addHelper16) Msgsize() (s int) {
return
}
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
func (z *interval16) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
@@ -546,7 +546,7 @@ func (z *interval16) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
func (z interval16) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 2
// write "start"
@@ -570,7 +570,7 @@ func (z interval16) EncodeMsg(en *msgp.Writer) (err error) {
return
}
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
func (z interval16) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 2
@@ -583,7 +583,7 @@ func (z interval16) MarshalMsg(b []byte) (o []byte, err error) {
return
}
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
func (z *interval16) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
@@ -621,13 +621,13 @@ func (z *interval16) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z interval16) Msgsize() (s int) {
s = 1 + 6 + msgp.Uint16Size + 5 + msgp.Uint16Size
return
}
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
func (z *runContainer16) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
@@ -701,7 +701,7 @@ func (z *runContainer16) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
func (z *runContainer16) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 2
// write "iv"
@@ -746,7 +746,7 @@ func (z *runContainer16) EncodeMsg(en *msgp.Writer) (err error) {
return
}
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
func (z *runContainer16) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 2
@@ -768,7 +768,7 @@ func (z *runContainer16) MarshalMsg(b []byte) (o []byte, err error) {
return
}
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
func (z *runContainer16) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
@@ -843,13 +843,13 @@ func (z *runContainer16) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *runContainer16) Msgsize() (s int) {
s = 1 + 3 + msgp.ArrayHeaderSize + (len(z.iv) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 5 + msgp.Int64Size
return
}
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
func (z *runIterator16) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
@@ -891,11 +891,6 @@ func (z *runIterator16) DecodeMsg(dc *msgp.Reader) (err error) {
if err != nil {
return
}
- case "curSeq":
- z.curSeq, err = dc.ReadInt64()
- if err != nil {
- return
- }
default:
err = dc.Skip()
if err != nil {
@@ -906,11 +901,11 @@ func (z *runIterator16) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
func (z *runIterator16) EncodeMsg(en *msgp.Writer) (err error) {
- // map header, size 4
+ // map header, size 3
// write "rc"
- err = en.Append(0x84, 0xa2, 0x72, 0x63)
+ err = en.Append(0x83, 0xa2, 0x72, 0x63)
if err != nil {
return err
}
@@ -943,24 +938,15 @@ func (z *runIterator16) EncodeMsg(en *msgp.Writer) (err error) {
if err != nil {
return
}
- // write "curSeq"
- err = en.Append(0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71)
- if err != nil {
- return err
- }
- err = en.WriteInt64(z.curSeq)
- if err != nil {
- return
- }
return
}
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
func (z *runIterator16) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
- // map header, size 4
+ // map header, size 3
// string "rc"
- o = append(o, 0x84, 0xa2, 0x72, 0x63)
+ o = append(o, 0x83, 0xa2, 0x72, 0x63)
if z.rc == nil {
o = msgp.AppendNil(o)
} else {
@@ -975,13 +961,10 @@ func (z *runIterator16) MarshalMsg(b []byte) (o []byte, err error) {
// string "curPosInIndex"
o = append(o, 0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78)
o = msgp.AppendUint16(o, z.curPosInIndex)
- // string "curSeq"
- o = append(o, 0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71)
- o = msgp.AppendInt64(o, z.curSeq)
return
}
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
func (z *runIterator16) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
@@ -1023,11 +1006,6 @@ func (z *runIterator16) UnmarshalMsg(bts []byte) (o []byte, err error) {
if err != nil {
return
}
- case "curSeq":
- z.curSeq, bts, err = msgp.ReadInt64Bytes(bts)
- if err != nil {
- return
- }
default:
bts, err = msgp.Skip(bts)
if err != nil {
@@ -1039,7 +1017,7 @@ func (z *runIterator16) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *runIterator16) Msgsize() (s int) {
s = 1 + 3
if z.rc == nil {
@@ -1047,11 +1025,11 @@ func (z *runIterator16) Msgsize() (s int) {
} else {
s += z.rc.Msgsize()
}
- s += 9 + msgp.Int64Size + 14 + msgp.Uint16Size + 7 + msgp.Int64Size
+ s += 9 + msgp.Int64Size + 14 + msgp.Uint16Size
return
}
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
func (z *uint16Slice) DecodeMsg(dc *msgp.Reader) (err error) {
var zjpj uint32
zjpj, err = dc.ReadArrayHeader()
@@ -1072,7 +1050,7 @@ func (z *uint16Slice) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
func (z uint16Slice) EncodeMsg(en *msgp.Writer) (err error) {
err = en.WriteArrayHeader(uint32(len(z)))
if err != nil {
@@ -1087,7 +1065,7 @@ func (z uint16Slice) EncodeMsg(en *msgp.Writer) (err error) {
return
}
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
func (z uint16Slice) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
o = msgp.AppendArrayHeader(o, uint32(len(z)))
@@ -1097,7 +1075,7 @@ func (z uint16Slice) MarshalMsg(b []byte) (o []byte, err error) {
return
}
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
func (z *uint16Slice) UnmarshalMsg(bts []byte) (o []byte, err error) {
var zgmo uint32
zgmo, bts, err = msgp.ReadArrayHeaderBytes(bts)
@@ -1119,7 +1097,7 @@ func (z *uint16Slice) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z uint16Slice) Msgsize() (s int) {
s = msgp.ArrayHeaderSize + (len(z) * (msgp.Uint16Size))
return
diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization.go b/vendor/github.com/RoaringBitmap/roaring/serialization.go
index 59c39a6630..7b7ed29b0a 100644
--- a/vendor/github.com/RoaringBitmap/roaring/serialization.go
+++ b/vendor/github.com/RoaringBitmap/roaring/serialization.go
@@ -2,8 +2,6 @@ package roaring
import (
"encoding/binary"
- "errors"
- "fmt"
"io"
"github.com/tinylib/msgp/msgp"
@@ -22,14 +20,6 @@ func (b *runContainer16) writeTo(stream io.Writer) (int, error) {
return stream.Write(buf)
}
-func (b *runContainer32) writeToMsgpack(stream io.Writer) (int, error) {
- bts, err := b.MarshalMsg(nil)
- if err != nil {
- return 0, err
- }
- return stream.Write(bts)
-}
-
func (b *runContainer16) writeToMsgpack(stream io.Writer) (int, error) {
bts, err := b.MarshalMsg(nil)
if err != nil {
@@ -38,46 +28,7 @@ func (b *runContainer16) writeToMsgpack(stream io.Writer) (int, error) {
return stream.Write(bts)
}
-func (b *runContainer32) readFromMsgpack(stream io.Reader) (int, error) {
- err := msgp.Decode(stream, b)
- return 0, err
-}
-
func (b *runContainer16) readFromMsgpack(stream io.Reader) (int, error) {
err := msgp.Decode(stream, b)
return 0, err
}
-
-var errCorruptedStream = errors.New("insufficient/odd number of stored bytes, corrupted stream detected")
-
-func (b *runContainer16) readFrom(stream io.Reader) (int, error) {
- b.iv = b.iv[:0]
- b.card = 0
- var numRuns uint16
- err := binary.Read(stream, binary.LittleEndian, &numRuns)
- if err != nil {
- return 0, err
- }
- nr := int(numRuns)
- encRun := make([]uint16, 2*nr)
- by := make([]byte, 4*nr)
- err = binary.Read(stream, binary.LittleEndian, &by)
- if err != nil {
- return 0, err
- }
- for i := range encRun {
- if len(by) < 2 {
- return 0, errCorruptedStream
- }
- encRun[i] = binary.LittleEndian.Uint16(by)
- by = by[2:]
- }
- for i := 0; i < nr; i++ {
- if i > 0 && b.iv[i-1].last() >= encRun[i*2] {
- return 0, fmt.Errorf("error: stored runContainer had runs that were not in sorted order!! (b.iv[i-1=%v].last = %v >= encRun[i=%v] = %v)", i-1, b.iv[i-1].last(), i, encRun[i*2])
- }
- b.iv = append(b.iv, interval16{start: encRun[i*2], length: encRun[i*2+1]})
- b.card += int64(encRun[i*2+1]) + 1
- }
- return 0, err
-}
diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go b/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go
index 7fcef7691b..4b9d9e3d48 100644
--- a/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go
+++ b/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go
@@ -4,6 +4,7 @@ package roaring
import (
"encoding/binary"
+ "errors"
"io"
)
@@ -26,6 +27,10 @@ func (b *arrayContainer) readFrom(stream io.Reader) (int, error) {
}
func (b *bitmapContainer) writeTo(stream io.Writer) (int, error) {
+ if b.cardinality <= arrayDefaultMaxSize {
+ return 0, errors.New("refusing to write bitmap container with cardinality of array container")
+ }
+
// Write set
buf := make([]byte, 8*len(b.bitmap))
for i, v := range b.bitmap {
@@ -69,6 +74,16 @@ func uint64SliceAsByteSlice(slice []uint64) []byte {
return by
}
+func uint16SliceAsByteSlice(slice []uint16) []byte {
+ by := make([]byte, len(slice)*2)
+
+ for i, v := range slice {
+ binary.LittleEndian.PutUint16(by[i*2:], v)
+ }
+
+ return by
+}
+
func byteSliceAsUint16Slice(slice []byte) []uint16 {
if len(slice)%2 != 0 {
panic("Slice size should be divisible by 2")
diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go
index c1d3ad3046..818a06c80b 100644
--- a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go
+++ b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go
@@ -3,8 +3,10 @@
package roaring
import (
+ "errors"
"io"
"reflect"
+ "runtime"
"unsafe"
)
@@ -14,26 +16,13 @@ func (ac *arrayContainer) writeTo(stream io.Writer) (int, error) {
}
func (bc *bitmapContainer) writeTo(stream io.Writer) (int, error) {
+ if bc.cardinality <= arrayDefaultMaxSize {
+ return 0, errors.New("refusing to write bitmap container with cardinality of array container")
+ }
buf := uint64SliceAsByteSlice(bc.bitmap)
return stream.Write(buf)
}
-// readFrom reads an arrayContainer from stream.
-// PRE-REQUISITE: you must size the arrayContainer correctly (allocate b.content)
-// *before* you call readFrom. We can't guess the size in the stream
-// by this point.
-func (ac *arrayContainer) readFrom(stream io.Reader) (int, error) {
- buf := uint16SliceAsByteSlice(ac.content)
- return io.ReadFull(stream, buf)
-}
-
-func (bc *bitmapContainer) readFrom(stream io.Reader) (int, error) {
- buf := uint64SliceAsByteSlice(bc.bitmap)
- n, err := io.ReadFull(stream, buf)
- bc.computeCardinality()
- return n, err
-}
-
func uint64SliceAsByteSlice(slice []uint64) []byte {
// make a new slice header
header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
@@ -42,8 +31,12 @@ func uint64SliceAsByteSlice(slice []uint64) []byte {
header.Len *= 8
header.Cap *= 8
+ // instantiate result and use KeepAlive so data isn't unmapped.
+ result := *(*[]byte)(unsafe.Pointer(&header))
+ runtime.KeepAlive(&slice)
+
// return it
- return *(*[]byte)(unsafe.Pointer(&header))
+ return result
}
func uint16SliceAsByteSlice(slice []uint16) []byte {
@@ -54,8 +47,12 @@ func uint16SliceAsByteSlice(slice []uint16) []byte {
header.Len *= 2
header.Cap *= 2
+ // instantiate result and use KeepAlive so data isn't unmapped.
+ result := *(*[]byte)(unsafe.Pointer(&header))
+ runtime.KeepAlive(&slice)
+
// return it
- return *(*[]byte)(unsafe.Pointer(&header))
+ return result
}
func (bc *bitmapContainer) asLittleEndianByteSlice() []byte {
@@ -64,50 +61,74 @@ func (bc *bitmapContainer) asLittleEndianByteSlice() []byte {
// Deserialization code follows
-func byteSliceAsUint16Slice(slice []byte) []uint16 {
+////
+// These methods (byteSliceAsUint16Slice,...) do not make copies,
+// they are pointer-based (unsafe). The caller is responsible to
+// ensure that the input slice does not get garbage collected, deleted
+// or modified while you hold the returned slince.
+////
+func byteSliceAsUint16Slice(slice []byte) (result []uint16) { // here we create a new slice holder
if len(slice)%2 != 0 {
panic("Slice size should be divisible by 2")
}
+ // reference: https://go101.org/article/unsafe.html
// make a new slice header
- header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
- // update its capacity and length
- header.Len /= 2
- header.Cap /= 2
+ // transfer the data from the given slice to a new variable (our result)
+ rHeader.Data = bHeader.Data
+ rHeader.Len = bHeader.Len / 2
+ rHeader.Cap = bHeader.Cap / 2
- // return it
- return *(*[]uint16)(unsafe.Pointer(&header))
+ // instantiate result and use KeepAlive so data isn't unmapped.
+ runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+ // return result
+ return
}
-func byteSliceAsUint64Slice(slice []byte) []uint64 {
+func byteSliceAsUint64Slice(slice []byte) (result []uint64) {
if len(slice)%8 != 0 {
panic("Slice size should be divisible by 8")
}
+ // reference: https://go101.org/article/unsafe.html
// make a new slice header
- header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
- // update its capacity and length
- header.Len /= 8
- header.Cap /= 8
+ // transfer the data from the given slice to a new variable (our result)
+ rHeader.Data = bHeader.Data
+ rHeader.Len = bHeader.Len / 8
+ rHeader.Cap = bHeader.Cap / 8
- // return it
- return *(*[]uint64)(unsafe.Pointer(&header))
+ // instantiate result and use KeepAlive so data isn't unmapped.
+ runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+ // return result
+ return
}
-func byteSliceAsInterval16Slice(slice []byte) []interval16 {
+func byteSliceAsInterval16Slice(slice []byte) (result []interval16) {
if len(slice)%4 != 0 {
panic("Slice size should be divisible by 4")
}
+ // reference: https://go101.org/article/unsafe.html
// make a new slice header
- header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
- // update its capacity and length
- header.Len /= 4
- header.Cap /= 4
+ // transfer the data from the given slice to a new variable (our result)
+ rHeader.Data = bHeader.Data
+ rHeader.Len = bHeader.Len / 4
+ rHeader.Cap = bHeader.Cap / 4
- // return it
- return *(*[]interval16)(unsafe.Pointer(&header))
+ // instantiate result and use KeepAlive so data isn't unmapped.
+ runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+ // return result
+ return
}
diff --git a/vendor/github.com/RoaringBitmap/roaring/shortiterator.go b/vendor/github.com/RoaringBitmap/roaring/shortiterator.go
index ef0acbd1ca..15b78bd0c1 100644
--- a/vendor/github.com/RoaringBitmap/roaring/shortiterator.go
+++ b/vendor/github.com/RoaringBitmap/roaring/shortiterator.go
@@ -5,6 +5,12 @@ type shortIterable interface {
next() uint16
}
+type shortPeekable interface {
+ shortIterable
+ peekNext() uint16
+ advanceIfNeeded(minval uint16)
+}
+
type shortIterator struct {
slice []uint16
loc int
@@ -19,3 +25,28 @@ func (si *shortIterator) next() uint16 {
si.loc++
return a
}
+
+func (si *shortIterator) peekNext() uint16 {
+ return si.slice[si.loc]
+}
+
+func (si *shortIterator) advanceIfNeeded(minval uint16) {
+ if si.hasNext() && si.peekNext() < minval {
+ si.loc = advanceUntil(si.slice, si.loc, len(si.slice), minval)
+ }
+}
+
+type reverseIterator struct {
+ slice []uint16
+ loc int
+}
+
+func (si *reverseIterator) hasNext() bool {
+ return si.loc >= 0
+}
+
+func (si *reverseIterator) next() uint16 {
+ a := si.slice[si.loc]
+ si.loc--
+ return a
+}
diff --git a/vendor/github.com/RoaringBitmap/roaring/util.go b/vendor/github.com/RoaringBitmap/roaring/util.go
index d212660d58..3a9a47236b 100644
--- a/vendor/github.com/RoaringBitmap/roaring/util.go
+++ b/vendor/github.com/RoaringBitmap/roaring/util.go
@@ -14,6 +14,17 @@ const (
serialCookie = 12347 // runs, arrays, and bitmaps
noOffsetThreshold = 4
+ // MaxUint32 is the largest uint32 value.
+ MaxUint32 = 4294967295
+
+ // MaxRange is One more than the maximum allowed bitmap bit index. For use as an upper
+ // bound for ranges.
+ MaxRange uint64 = MaxUint32 + 1
+
+ // MaxUint16 is the largest 16 bit unsigned int.
+ // This is the largest value an interval16 can store.
+ MaxUint16 = 65535
+
// Compute wordSizeInBytes, the size of a word in bytes.
_m = ^uint64(0)
_logS = _m>>8&1 + _m>>16&1 + _m>>32&1
@@ -114,7 +125,6 @@ func flipBitmapRange(bitmap []uint64, start int, end int) {
endword := (end - 1) / 64
bitmap[firstword] ^= ^(^uint64(0) << uint(start%64))
for i := firstword; i < endword; i++ {
- //p("flipBitmapRange on i=%v", i)
bitmap[i] = ^bitmap[i]
}
bitmap[endword] ^= ^uint64(0) >> (uint(-end) % 64)
@@ -292,24 +302,3 @@ func minOfUint16(a, b uint16) uint16 {
}
return b
}
-
-func maxInt(a, b int) int {
- if a > b {
- return a
- }
- return b
-}
-
-func maxUint16(a, b uint16) uint16 {
- if a > b {
- return a
- }
- return b
-}
-
-func minUint16(a, b uint16) uint16 {
- if a < b {
- return a
- }
- return b
-}