MB-57871, MB-62230: Upgrade to zapx/[email protected] + [email protected] (#…

…2088) MB-62230: * Add elements to bitmap in a batch. * 5f8f0f0 Aditi Ahuja | MB-62230 - Pre-filtering performance optimisations MB-57871: * 7bab2f9 Abhi Dangeti | MB-57871: One more nil check before dereferencing postings * bf971e6 Abhi Dangeti | Add nil pointer checks for postings before dereference * 9a80f5f Abhi Dangeti | One more case for zero chunk size / mode * 0c6f397 Mohd Shaad Khan | Handle zero chunk size --------- Co-authored-by: Abhinav Dangeti <[email protected]>
blevesearch · Oct 18, 2024 · ab10172 · ab10172
1 parent 02d37a4
commit ab10172
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 11 deletions.
diff --git a/go.mod b/go.mod
@@ -7,7 +7,7 @@ require (
 	github.com/bits-and-blooms/bitset v1.12.0
 	github.com/blevesearch/bleve_index_api v1.1.12
 	github.com/blevesearch/geo v0.1.20
-	github.com/blevesearch/go-faiss v1.0.22
+	github.com/blevesearch/go-faiss v1.0.23
 	github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475
 	github.com/blevesearch/go-porterstemmer v1.0.3
 	github.com/blevesearch/goleveldb v1.0.1
@@ -24,7 +24,7 @@ require (
 	github.com/blevesearch/zapx/v13 v13.3.10
 	github.com/blevesearch/zapx/v14 v14.3.10
 	github.com/blevesearch/zapx/v15 v15.3.15
-	github.com/blevesearch/zapx/v16 v16.1.7-0.20241016180148-7bab2f98bc3d
+	github.com/blevesearch/zapx/v16 v16.1.7
 	github.com/couchbase/moss v0.2.0
 	github.com/golang/protobuf v1.3.2
 	github.com/spf13/cobra v1.7.0

diff --git a/go.sum b/go.sum
@@ -6,8 +6,8 @@ github.com/blevesearch/bleve_index_api v1.1.12 h1:P4bw9/G/5rulOF7SJ9l4FsDoo7UFJ+
 github.com/blevesearch/bleve_index_api v1.1.12/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8=
 github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM=
 github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w=
-github.com/blevesearch/go-faiss v1.0.22 h1:j6jwgCOy2a2EQUTOYxjBA59rMn5KPA0jbfYyHNgc2Ls=
-github.com/blevesearch/go-faiss v1.0.22/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk=
+github.com/blevesearch/go-faiss v1.0.23 h1:Wmc5AFwDLKGl2L6mjLX1Da3vCL0EKa2uHHSorcIS1Uc=
+github.com/blevesearch/go-faiss v1.0.23/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk=
 github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:kDy+zgJFJJoJYBvdfBSiZYBbdsUL0XcjHYWezpQBGPA=
 github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A=
 github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
@@ -43,8 +43,8 @@ github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz7
 github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
 github.com/blevesearch/zapx/v15 v15.3.15 h1:JydcGIq279tmTrfBBSPDF/VOiCMBLQ+rJulTXrGFlGA=
 github.com/blevesearch/zapx/v15 v15.3.15/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
-github.com/blevesearch/zapx/v16 v16.1.7-0.20241016180148-7bab2f98bc3d h1:XpLXi4MVxygh7yQaMfSt/ta7TgfaCYD6pBUfk+FfaHo=
-github.com/blevesearch/zapx/v16 v16.1.7-0.20241016180148-7bab2f98bc3d/go.mod h1:I1CnZUekZFlJZ+cUCnmoJZTD49suuunjuH6BdOTtT9g=
+github.com/blevesearch/zapx/v16 v16.1.7 h1:I07qV6l1rPda19zyof9Q2J9E8cjZ57pQhNY0+ePI5vM=
+github.com/blevesearch/zapx/v16 v16.1.7/go.mod h1:JqQlOqlRVaYDkpLIl3JnKql8u4zKTNlVEa3nLsi0Gn8=
 github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
 github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
 github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o=

diff --git a/index/scorch/optimize_knn.go b/index/scorch/optimize_knn.go
@@ -115,7 +115,7 @@ func (o *OptimizeVR) Finish() error {
 							}
 
 							eligibleLocalDocNums := make([]uint64,
-								eligibleVectorInternalIDs.Stats().Cardinality)
+								eligibleVectorInternalIDs.GetCardinality())
 							// get the (segment-)local document numbers
 							for i, docNum := range eligibleVectorInternalIDs.ToArray() {
 								localDocNum := o.snapshot.localDocNumFromGlobal(index,

diff --git a/index/scorch/snapshot_index_vr.go b/index/scorch/snapshot_index_vr.go
@@ -64,10 +64,17 @@ type IndexSnapshotVectorReader struct {
 // (AND) operations. Eg. finding the eligible doc IDs present in a segment.
 func (i *IndexSnapshotVectorReader) getEligibleDocIDs() *roaring.Bitmap {
 	res := roaring.NewBitmap()
-	// converts the doc IDs to uint32 and returns
-	for _, eligibleDocInternalID := range i.eligibleDocIDs {
-		internalDocID, _ := docInternalToNumber(index.IndexInternalID(eligibleDocInternalID))
-		res.Add(uint32(internalDocID))
+	if len(i.eligibleDocIDs) > 0 {
+		internalDocIDs := make([]uint32, 0, len(i.eligibleDocIDs))
+		// converts the doc IDs to uint32 and returns
+		for _, eligibleDocInternalID := range i.eligibleDocIDs {
+			internalDocID, err := docInternalToNumber(index.IndexInternalID(eligibleDocInternalID))
+			if err != nil {
+				continue
+			}
+			internalDocIDs = append(internalDocIDs, uint32(internalDocID))
+		}
+		res.AddMany(internalDocIDs)
 	}
 	return res
 }