Skip to content

Commit

Permalink
MB-64604: [BP] Fix interpreting scorch config: "fieldTFRCacheThreshold"
Browse files Browse the repository at this point in the history
+ Setting the default to 0 on account of the panics caught in the MB.
+ Firstly, refactor FieldTFRCacheThreshold to fieldTFRCacheThreshold
  for some naming consistency here.
+ This threshold can be used to toggle recycling of TermFieldReaders on/off.
+ Couchbase users will have the ability to provide this setting within a
  JSON payload, which when interpreted into a map[string]interface{} will
  need to be interpreted as a float64.
+ Should library users set it as an int within the index config - we'll
  honor that setting as well.
  • Loading branch information
abhinavdangeti committed Dec 17, 2024
1 parent 9ada1ca commit c86ffd0
Showing 1 changed file with 21 additions and 13 deletions.
34 changes: 21 additions & 13 deletions index/scorch/snapshot_index.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,15 +51,6 @@ type asynchSegmentResult struct {

var reflectStaticSizeIndexSnapshot int

// DefaultFieldTFRCacheThreshold limits the number of TermFieldReaders(TFR) for
// a field in an index snapshot. Without this limit, when recycling TFRs, it is
// possible that a very large number of TFRs may be added to the recycle
// cache, which could eventually lead to significant memory consumption.
// This threshold can be overwritten by users at the library level by changing the
// exported variable, or at the index level by setting the FieldTFRCacheThreshold
// in the kvConfig.
var DefaultFieldTFRCacheThreshold uint64 = 10

func init() {
var is interface{} = IndexSnapshot{}
reflectStaticSizeIndexSnapshot = int(reflect.TypeOf(is).Size())
Expand Down Expand Up @@ -567,10 +558,27 @@ func (i *IndexSnapshot) allocTermFieldReaderDicts(field string) (tfr *IndexSnaps
}
}

func (i *IndexSnapshot) getFieldTFRCacheThreshold() uint64 {
// DefaultFieldTFRCacheThreshold limits the number of TermFieldReaders(TFR) for
// a field in an index snapshot. Without this limit, when recycling TFRs, it is
// possible that a very large number of TFRs may be added to the recycle
// cache, which could eventually lead to significant memory consumption.
// This threshold can be overwritten by users at the library level by changing the
// exported variable, or at the index level by setting the "fieldTFRCacheThreshold"
// in the kvConfig.
var DefaultFieldTFRCacheThreshold int = 0 // disabled because it causes MB-64604

func (i *IndexSnapshot) getFieldTFRCacheThreshold() int {
if i.parent.config != nil {
if _, ok := i.parent.config["FieldTFRCacheThreshold"]; ok {
return i.parent.config["FieldTFRCacheThreshold"].(uint64)
if _, exists := i.parent.config["fieldTFRCacheThreshold"]; exists {
val := i.parent.config["fieldTFRCacheThreshold"]
if x, ok := val.(float64); ok {
// JSON unmarshal-ed into a map[string]interface{} will default
// to float64 for numbers, so we need to check for float64 first.
return int(x)
} else if x, ok := val.(int); ok {
// If library users provided an int in the config, we'll honor it.
return x
}
}
}
return DefaultFieldTFRCacheThreshold
Expand All @@ -597,7 +605,7 @@ func (i *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReader
if i.fieldTFRs == nil {
i.fieldTFRs = map[string][]*IndexSnapshotTermFieldReader{}
}
if uint64(len(i.fieldTFRs[tfr.field])) < i.getFieldTFRCacheThreshold() {
if len(i.fieldTFRs[tfr.field]) < i.getFieldTFRCacheThreshold() {
i.fieldTFRs[tfr.field] = append(i.fieldTFRs[tfr.field], tfr)
}
i.m2.Unlock()
Expand Down

0 comments on commit c86ffd0

Please sign in to comment.