Skip to content

Commit

Permalink
MB-64604: Merge remote-tracking branch 'origin/trinity-couchbase' int…
Browse files Browse the repository at this point in the history 
…o 'master'

|\
| * 5c53634 Abhi Dangeti | MB-64604: Remove unnecessary second map lookup (#2121)
| * 78cf789 Abhi Dangeti | MB-64604: Fix interpreting scorch config: "fieldTFRCacheThreshold" (#2117)
| * 7d627b9 Aditi Ahuja | MB-64360 - Upgrade zapx v16 (#2107)
  • Loading branch information
@abhinavdangeti
abhinavdangeti committed Dec 17, 2024
2 parents ff6ba91 + 5c53634 commit 8cb6b89
Show file tree
Hide file tree
Showing 3 changed files with 43 additions and 29 deletions.
7 changes: 7 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,17 @@ github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIq
github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk=
github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU=
github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
<<<<<<< HEAD

Check failure on line 44 in go.sum

View workflow job for this annotation

GitHub Actions / test (1.22.x, ubuntu-latest) 

wrong number of fields 2

Check failure on line 44 in go.sum

View workflow job for this annotation

GitHub Actions / test (1.22.x, macos-latest) 

wrong number of fields 2
github.com/blevesearch/zapx/v15 v15.3.17 h1:NkkMI98pYLq/uHnB6YWcITrrLpCVyvZ9iP+AyfpW1Ys=
github.com/blevesearch/zapx/v15 v15.3.17/go.mod h1:vXRQzJJvlGVCdmOD5hg7t7JdjUT5DmDPhsAfjvtzIq8=
github.com/blevesearch/zapx/v16 v16.1.10 h1:moxlODQYuwqsXWK7Yyj40Wr1G8A4QKB32+fz3OLlEDU=
github.com/blevesearch/zapx/v16 v16.1.10/go.mod h1:Xtloe2uqSXH2j3yrQr9yGiHwz3Hz/fpHn5szTqpyizs=
=======
github.com/blevesearch/zapx/v15 v15.3.16 h1:Ct3rv7FUJPfPk99TI/OofdC+Kpb4IdyfdMH48sb+FmE=
github.com/blevesearch/zapx/v15 v15.3.16/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
github.com/blevesearch/zapx/v16 v16.1.9-0.20241128100831-c85f4c5695fd h1:Es5x1U5ii+lM/zStrb4YCY4R1sKVEi0d6XAOZQ5Q0N4=
github.com/blevesearch/zapx/v16 v16.1.9-0.20241128100831-c85f4c5695fd/go.mod h1:JqQlOqlRVaYDkpLIl3JnKql8u4zKTNlVEa3nLsi0Gn8=
>>>>>>> origin/trinity-couchbase
github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o=
Expand Down
33 changes: 20 additions & 13 deletions index/scorch/snapshot_index.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,15 +52,6 @@ type asynchSegmentResult struct {

var reflectStaticSizeIndexSnapshot int

// DefaultFieldTFRCacheThreshold limits the number of TermFieldReaders(TFR) for
// a field in an index snapshot. Without this limit, when recycling TFRs, it is
// possible that a very large number of TFRs may be added to the recycle
// cache, which could eventually lead to significant memory consumption.
// This threshold can be overwritten by users at the library level by changing the
// exported variable, or at the index level by setting the FieldTFRCacheThreshold
// in the kvConfig.
var DefaultFieldTFRCacheThreshold uint64 = 10

func init() {
var is interface{} = IndexSnapshot{}
reflectStaticSizeIndexSnapshot = int(reflect.TypeOf(is).Size())
Expand Down Expand Up @@ -640,10 +631,26 @@ func (is *IndexSnapshot) allocTermFieldReaderDicts(field string) (tfr *IndexSnap
}
}

func (is *IndexSnapshot) getFieldTFRCacheThreshold() uint64 {
// DefaultFieldTFRCacheThreshold limits the number of TermFieldReaders(TFR) for
// a field in an index snapshot. Without this limit, when recycling TFRs, it is
// possible that a very large number of TFRs may be added to the recycle
// cache, which could eventually lead to significant memory consumption.
// This threshold can be overwritten by users at the library level by changing the
// exported variable, or at the index level by setting the "fieldTFRCacheThreshold"
// in the kvConfig.
var DefaultFieldTFRCacheThreshold int = 0 // disabled because it causes MB-64604

func (is *IndexSnapshot) getFieldTFRCacheThreshold() int {
if is.parent.config != nil {
if _, ok := is.parent.config["FieldTFRCacheThreshold"]; ok {
return is.parent.config["FieldTFRCacheThreshold"].(uint64)
if val, exists := is.parent.config["fieldTFRCacheThreshold"]; exists {
if x, ok := val.(float64); ok {
// JSON unmarshal-ed into a map[string]interface{} will default
// to float64 for numbers, so we need to check for float64 first.
return int(x)
} else if x, ok := val.(int); ok {
// If library users provided an int in the config, we'll honor it.
return x
}
}
}
return DefaultFieldTFRCacheThreshold
Expand All @@ -670,7 +677,7 @@ func (is *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReade
if is.fieldTFRs == nil {
is.fieldTFRs = map[string][]*IndexSnapshotTermFieldReader{}
}
if uint64(len(is.fieldTFRs[tfr.field])) < is.getFieldTFRCacheThreshold() {
if len(is.fieldTFRs[tfr.field]) < is.getFieldTFRCacheThreshold() {
tfr.bytesRead = 0
is.fieldTFRs[tfr.field] = append(is.fieldTFRs[tfr.field], tfr)
}
Expand Down
32 changes: 16 additions & 16 deletions index_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,8 +421,8 @@ func TestBytesRead(t *testing.T) {
}
stats, _ = idx.StatsMap()["index"].(map[string]interface{})
bytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64)
if bytesRead-prevBytesRead != 23 && res.Cost == bytesRead-prevBytesRead {
t.Fatalf("expected bytes read for query string 23, got %v",
if bytesRead-prevBytesRead != 66 && res.Cost == bytesRead-prevBytesRead {
t.Fatalf("expected bytes read for query string 66, got %v",
bytesRead-prevBytesRead)
}
prevBytesRead = bytesRead
Expand Down Expand Up @@ -454,8 +454,8 @@ func TestBytesRead(t *testing.T) {

stats, _ = idx.StatsMap()["index"].(map[string]interface{})
bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64)
if !approxSame(bytesRead-prevBytesRead, 150) && res.Cost == bytesRead-prevBytesRead {
t.Fatalf("expected bytes read for faceted query is around 150, got %v",
if !approxSame(bytesRead-prevBytesRead, 196) && res.Cost == bytesRead-prevBytesRead {
t.Fatalf("expected bytes read for faceted query is around 196, got %v",
bytesRead-prevBytesRead)
}
prevBytesRead = bytesRead
Expand Down Expand Up @@ -487,8 +487,8 @@ func TestBytesRead(t *testing.T) {

stats, _ = idx.StatsMap()["index"].(map[string]interface{})
bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64)
if bytesRead-prevBytesRead != 60 && res.Cost == bytesRead-prevBytesRead {
t.Fatalf("expected bytes read for query with highlighter is 60, got %v",
if bytesRead-prevBytesRead != 105 && res.Cost == bytesRead-prevBytesRead {
t.Fatalf("expected bytes read for query with highlighter is 105, got %v",
bytesRead-prevBytesRead)
}
prevBytesRead = bytesRead
Expand All @@ -504,8 +504,8 @@ func TestBytesRead(t *testing.T) {
// since it's created afresh and not reused
stats, _ = idx.StatsMap()["index"].(map[string]interface{})
bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64)
if bytesRead-prevBytesRead != 83 && res.Cost == bytesRead-prevBytesRead {
t.Fatalf("expected bytes read for disjunction query is 83, got %v",
if bytesRead-prevBytesRead != 120 && res.Cost == bytesRead-prevBytesRead {
t.Fatalf("expected bytes read for disjunction query is 120, got %v",
bytesRead-prevBytesRead)
}
}
Expand Down Expand Up @@ -577,8 +577,8 @@ func TestBytesReadStored(t *testing.T) {
}
stats, _ = idx.StatsMap()["index"].(map[string]interface{})
bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64)
if bytesRead-prevBytesRead != 15 && bytesRead-prevBytesRead == res.Cost {
t.Fatalf("expected the bytes read stat to be around 15, got %v", bytesRead-prevBytesRead)
if bytesRead-prevBytesRead != 48 && bytesRead-prevBytesRead == res.Cost {
t.Fatalf("expected the bytes read stat to be around 48, got %v", bytesRead-prevBytesRead)
}
prevBytesRead = bytesRead

Expand All @@ -592,8 +592,8 @@ func TestBytesReadStored(t *testing.T) {
stats, _ = idx.StatsMap()["index"].(map[string]interface{})
bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64)

if bytesRead-prevBytesRead != 26478 && bytesRead-prevBytesRead == res.Cost {
t.Fatalf("expected the bytes read stat to be around 26478, got %v",
if bytesRead-prevBytesRead != 26511 && bytesRead-prevBytesRead == res.Cost {
t.Fatalf("expected the bytes read stat to be around 26511, got %v",
bytesRead-prevBytesRead)
}
idx.Close()
Expand Down Expand Up @@ -653,8 +653,8 @@ func TestBytesReadStored(t *testing.T) {
}
stats, _ = idx1.StatsMap()["index"].(map[string]interface{})
bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64)
if bytesRead-prevBytesRead != 12 && bytesRead-prevBytesRead == res.Cost {
t.Fatalf("expected the bytes read stat to be around 12, got %v", bytesRead-prevBytesRead)
if bytesRead-prevBytesRead != 47 && bytesRead-prevBytesRead == res.Cost {
t.Fatalf("expected the bytes read stat to be around 47, got %v", bytesRead-prevBytesRead)
}
prevBytesRead = bytesRead

Expand All @@ -666,8 +666,8 @@ func TestBytesReadStored(t *testing.T) {

stats, _ = idx1.StatsMap()["index"].(map[string]interface{})
bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64)
if bytesRead-prevBytesRead != 42 && bytesRead-prevBytesRead == res.Cost {
t.Fatalf("expected the bytes read stat to be around 42, got %v", bytesRead-prevBytesRead)
if bytesRead-prevBytesRead != 77 && bytesRead-prevBytesRead == res.Cost {
t.Fatalf("expected the bytes read stat to be around 77, got %v", bytesRead-prevBytesRead)
}
}

Expand Down

0 comments on commit 8cb6b89

Please sign in to comment.