diff --git a/go.mod b/go.mod index ff4aef01ae6..7dd2c275073 100644 --- a/go.mod +++ b/go.mod @@ -248,7 +248,7 @@ require ( github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common/sigv4 v0.1.0 // indirect - github.com/prometheus/exporter-toolkit v0.13.0 // indirect + github.com/prometheus/exporter-toolkit v0.13.1 // indirect github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect github.com/rs/cors v1.11.0 // indirect github.com/rs/xid v1.6.0 // indirect @@ -284,7 +284,7 @@ require ( ) // Using a fork of Prometheus with Mimir-specific changes. -replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20241202100552-120c71abae28 +replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20241203165959-320e9aa3985d // Replace memberlist with our fork which includes some fixes that haven't been // merged upstream yet: diff --git a/go.sum b/go.sum index 5b3be74afd2..5b0c15dcecc 100644 --- a/go.sum +++ b/go.sum @@ -1279,8 +1279,8 @@ github.com/grafana/gomemcache v0.0.0-20241016125027-0a5bcc5aef40 h1:1TeKhyS+pvzO github.com/grafana/gomemcache v0.0.0-20241016125027-0a5bcc5aef40/go.mod h1:IGRj8oOoxwJbHBYl1+OhS9UjQR0dv6SQOep7HqmtyFU= github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe h1:yIXAAbLswn7VNWBIvM71O2QsgfgW9fRXZNR0DXe6pDU= github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/grafana/mimir-prometheus v0.0.0-20241202100552-120c71abae28 h1:aZcCAcjvHFNcpqnWOpdCjvKqA1jN1syHKYxFFrQBYYc= -github.com/grafana/mimir-prometheus v0.0.0-20241202100552-120c71abae28/go.mod h1:zJ8WD0NpeWRafBVPm243oe6z1PFcQGEsX71fIwVsycs= +github.com/grafana/mimir-prometheus v0.0.0-20241203165959-320e9aa3985d h1:ia2JbmaZA2VUP9HoE5wDjNmR3ZyJYH1yAodg/KoX/IY= +github.com/grafana/mimir-prometheus v0.0.0-20241203165959-320e9aa3985d/go.mod h1:SLFE244uZpIWNevAuaMLJmsy5QluDTSUwLKKGK3jkk0= github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956 h1:em1oddjXL8c1tL0iFdtVtPloq2hRPen2MJQKoAWpxu0= github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= github.com/grafana/prometheus-alertmanager v0.25.1-0.20240930132144-b5e64e81e8d3 h1:6D2gGAwyQBElSrp3E+9lSr7k8gLuP3Aiy20rweLWeBw= @@ -1615,8 +1615,8 @@ github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPA github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4= github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI= -github.com/prometheus/exporter-toolkit v0.13.0 h1:lmA0Q+8IaXgmFRKw09RldZmZdnvu9wwcDLIXGmTPw1c= -github.com/prometheus/exporter-toolkit v0.13.0/go.mod h1:2uop99EZl80KdXhv/MxVI2181fMcwlsumFOqBecGkG0= +github.com/prometheus/exporter-toolkit v0.13.1 h1:Evsh0gWQo2bdOHlnz9+0Nm7/OFfIwhE2Ws4A2jIlR04= +github.com/prometheus/exporter-toolkit v0.13.1/go.mod h1:ujdv2YIOxtdFxxqtloLpbqmxd5J0Le6IITUvIRSWjj0= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= @@ -1791,8 +1791,8 @@ go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= -go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= -go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= go.opentelemetry.io/otel/sdk/metric v1.30.0 h1:QJLT8Pe11jyHBHfSAgYH7kEmT24eX792jZO1bo4BXkM= go.opentelemetry.io/otel/sdk/metric v1.30.0/go.mod h1:waS6P3YqFNzeP01kuo/MBBYqaoBJl7efRQHOaydhy1Y= go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= diff --git a/pkg/distributor/otel.go b/pkg/distributor/otel.go index b4754fb85c9..2055e7544d4 100644 --- a/pkg/distributor/otel.go +++ b/pkg/distributor/otel.go @@ -394,8 +394,9 @@ func otelMetricsToMetadata(addSuffixes bool, md pmetric.Metrics) []*mimirpb.Metr for k := 0; k < scopeMetrics.Metrics().Len(); k++ { metric := scopeMetrics.Metrics().At(k) entry := mimirpb.MetricMetadata{ - Type: otelMetricTypeToMimirMetricType(metric), - MetricFamilyName: prometheustranslator.BuildCompliantName(metric, "", addSuffixes), + Type: otelMetricTypeToMimirMetricType(metric), + // TODO(krajorama): when UTF-8 is configurable from user limits, replace "false" appropriately. + MetricFamilyName: prometheustranslator.BuildCompliantName(metric, "", addSuffixes, false), Help: metric.Description(), Unit: metric.Unit(), } diff --git a/pkg/distributor/otel_test.go b/pkg/distributor/otel_test.go index 0f0faa65a2d..deffa2d6ae7 100644 --- a/pkg/distributor/otel_test.go +++ b/pkg/distributor/otel_test.go @@ -75,7 +75,7 @@ func TestOTelMetricsToTimeSeries(t *testing.T) { // Ignore, as these labels are generated from service.name and service.instance.id continue default: - k = prometheustranslator.NormalizeLabel(k) + k = prometheustranslator.NormalizeLabel(k, false) } expTargetInfoLabels = append(expTargetInfoLabels, mimirpb.LabelAdapter{ Name: k, diff --git a/pkg/distributor/otlp/helper_generated.go b/pkg/distributor/otlp/helper_generated.go index e65b83f1679..5bbbcab93dc 100644 --- a/pkg/distributor/otlp/helper_generated.go +++ b/pkg/distributor/otlp/helper_generated.go @@ -160,7 +160,7 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, setting // map ensures no duplicate label names. l := make(map[string]string, maxLabelCount) for _, label := range labels { - var finalKey = prometheustranslator.NormalizeLabel(label.Name) + var finalKey = prometheustranslator.NormalizeLabel(label.Name, settings.AllowUTF8) if existingValue, alreadyExists := l[finalKey]; alreadyExists { l[finalKey] = existingValue + ";" + label.Value } else { @@ -169,7 +169,7 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, setting } for _, lbl := range promotedAttrs { - normalized := prometheustranslator.NormalizeLabel(lbl.Name) + normalized := prometheustranslator.NormalizeLabel(lbl.Name, settings.AllowUTF8) if _, exists := l[normalized]; !exists { l[normalized] = lbl.Value } @@ -208,7 +208,7 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, setting } // internal labels should be maintained if !(len(name) > 4 && name[:2] == "__" && name[len(name)-2:] == "__") { - name = prometheustranslator.NormalizeLabel(name) + name = prometheustranslator.NormalizeLabel(name, settings.AllowUTF8) } l[name] = extras[i+1] } diff --git a/pkg/distributor/otlp/metrics_to_prw_generated.go b/pkg/distributor/otlp/metrics_to_prw_generated.go index ea8526657e6..4aeafeb11ce 100644 --- a/pkg/distributor/otlp/metrics_to_prw_generated.go +++ b/pkg/distributor/otlp/metrics_to_prw_generated.go @@ -38,13 +38,14 @@ import ( ) type Settings struct { - ExternalLabels map[string]string Namespace string - PromoteResourceAttributes []string + ExternalLabels map[string]string DisableTargetInfo bool ExportCreatedMetric bool AddMetricSuffixes bool SendMetadata bool + AllowUTF8 bool + PromoteResourceAttributes []string EnableCreatedTimestampZeroIngestion bool ValidIntervalCreatedTimestampZeroIngestion time.Duration } @@ -98,7 +99,7 @@ func (c *MimirConverter) FromMetrics(ctx context.Context, md pmetric.Metrics, se continue } - promName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) + promName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes, settings.AllowUTF8) // handle individual metrics based on type //exhaustive:enforce diff --git a/pkg/distributor/otlp/otlp_to_openmetrics_metadata_generated.go b/pkg/distributor/otlp/otlp_to_openmetrics_metadata_generated.go index e9d779b6ab2..2c46a2ecf56 100644 --- a/pkg/distributor/otlp/otlp_to_openmetrics_metadata_generated.go +++ b/pkg/distributor/otlp/otlp_to_openmetrics_metadata_generated.go @@ -46,7 +46,7 @@ func otelMetricTypeToPromMetricType(otelMetric pmetric.Metric) mimirpb.MetricMet return mimirpb.UNKNOWN } -func OtelMetricsToMetadata(md pmetric.Metrics, addMetricSuffixes bool) []*mimirpb.MetricMetadata { +func OtelMetricsToMetadata(md pmetric.Metrics, addMetricSuffixes, allowUTF8 bool) []*mimirpb.MetricMetadata { resourceMetricsSlice := md.ResourceMetrics() metadataLength := 0 @@ -68,7 +68,7 @@ func OtelMetricsToMetadata(md pmetric.Metrics, addMetricSuffixes bool) []*mimirp metric := scopeMetrics.Metrics().At(k) entry := mimirpb.MetricMetadata{ Type: otelMetricTypeToPromMetricType(metric), - MetricFamilyName: prometheustranslator.BuildCompliantName(metric, "", addMetricSuffixes), + MetricFamilyName: prometheustranslator.BuildCompliantName(metric, "", addMetricSuffixes, allowUTF8), Help: metric.Description(), } metadata = append(metadata, &entry) diff --git a/vendor/github.com/prometheus/exporter-toolkit/web/landing_page.go b/vendor/github.com/prometheus/exporter-toolkit/web/landing_page.go index 68266213edd..d417c15e0d2 100644 --- a/vendor/github.com/prometheus/exporter-toolkit/web/landing_page.go +++ b/vendor/github.com/prometheus/exporter-toolkit/web/landing_page.go @@ -106,6 +106,10 @@ func NewLandingPage(c LandingConfig) (*LandingPageHandler, error) { } func (h *LandingPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } w.Header().Add("Content-Type", "text/html; charset=UTF-8") w.Write(h.landingPage) } diff --git a/vendor/github.com/prometheus/prometheus/config/config.go b/vendor/github.com/prometheus/prometheus/config/config.go index 7fb77b0e6b7..ef3ea5e67ed 100644 --- a/vendor/github.com/prometheus/prometheus/config/config.go +++ b/vendor/github.com/prometheus/prometheus/config/config.go @@ -106,6 +106,18 @@ func Load(s string, logger *slog.Logger) (*Config, error) { if !b.Labels().IsEmpty() { cfg.GlobalConfig.ExternalLabels = b.Labels() } + + switch cfg.OTLPConfig.TranslationStrategy { + case UnderscoreEscapingWithSuffixes: + case "": + case NoUTF8EscapingWithSuffixes: + if cfg.GlobalConfig.MetricNameValidationScheme == LegacyValidationConfig { + return nil, errors.New("OTLP translation strategy NoUTF8EscapingWithSuffixes is not allowed when UTF8 is disabled") + } + default: + return nil, fmt.Errorf("unsupported OTLP translation strategy %q", cfg.OTLPConfig.TranslationStrategy) + } + return cfg, nil } @@ -239,7 +251,9 @@ var ( } // DefaultOTLPConfig is the default OTLP configuration. - DefaultOTLPConfig = OTLPConfig{} + DefaultOTLPConfig = OTLPConfig{ + TranslationStrategy: UnderscoreEscapingWithSuffixes, + } ) // Config is the top-level configuration for Prometheus's config files. @@ -1402,9 +1416,20 @@ func getGoGCEnv() int { return DefaultRuntimeConfig.GoGC } +type translationStrategyOption string + +var ( + // NoUTF8EscapingWithSuffixes will keep UTF-8 characters as they are, units and type suffixes will still be added. + NoUTF8EscapingWithSuffixes translationStrategyOption = "NoUTF8EscapingWithSuffixes" + // UnderscoreEscapingWithSuffixes is the default option for translating OTLP to Prometheus. + // This option will translate all UTF-8 characters to underscores, while adding units and type suffixes. + UnderscoreEscapingWithSuffixes translationStrategyOption = "UnderscoreEscapingWithSuffixes" +) + // OTLPConfig is the configuration for writing to the OTLP endpoint. type OTLPConfig struct { - PromoteResourceAttributes []string `yaml:"promote_resource_attributes,omitempty"` + PromoteResourceAttributes []string `yaml:"promote_resource_attributes,omitempty"` + TranslationStrategy translationStrategyOption `yaml:"translation_strategy,omitempty"` } // UnmarshalYAML implements the yaml.Unmarshaler interface. diff --git a/vendor/github.com/prometheus/prometheus/model/textparse/nhcbparse.go b/vendor/github.com/prometheus/prometheus/model/textparse/nhcbparse.go index d019c327c37..6fe2e8e54eb 100644 --- a/vendor/github.com/prometheus/prometheus/model/textparse/nhcbparse.go +++ b/vendor/github.com/prometheus/prometheus/model/textparse/nhcbparse.go @@ -283,18 +283,18 @@ func (p *NHCBParser) handleClassicHistogramSeries(lset labels.Labels) bool { le, err := strconv.ParseFloat(lset.Get(labels.BucketLabel), 64) if err == nil && !math.IsNaN(le) { p.processClassicHistogramSeries(lset, "_bucket", func(hist *convertnhcb.TempHistogram) { - hist.BucketCounts[le] = p.value + _ = hist.SetBucketCount(le, p.value) }) return true } case strings.HasSuffix(mName, "_count"): p.processClassicHistogramSeries(lset, "_count", func(hist *convertnhcb.TempHistogram) { - hist.Count = p.value + _ = hist.SetCount(p.value) }) return true case strings.HasSuffix(mName, "_sum"): p.processClassicHistogramSeries(lset, "_sum", func(hist *convertnhcb.TempHistogram) { - hist.Sum = p.value + _ = hist.SetSum(p.value) }) return true } @@ -306,8 +306,8 @@ func (p *NHCBParser) processClassicHistogramSeries(lset labels.Labels, suffix st p.storeClassicLabels() p.tempCT = p.parser.CreatedTimestamp() p.state = stateCollecting + p.tempLsetNHCB = convertnhcb.GetHistogramMetricBase(lset, suffix) } - p.tempLsetNHCB = convertnhcb.GetHistogramMetricBase(lset, suffix) p.storeExemplars() updateHist(&p.tempNHCB) } @@ -335,7 +335,6 @@ func (p *NHCBParser) nextExemplarPtr() *exemplar.Exemplar { func (p *NHCBParser) swapExemplars() { p.exemplars = p.tempExemplars[:p.tempExemplarCount] p.tempExemplars = p.tempExemplars[:0] - p.tempExemplarCount = 0 } // processNHCB converts the collated classic histogram series to NHCB and caches the info @@ -344,33 +343,32 @@ func (p *NHCBParser) processNHCB() bool { if p.state != stateCollecting { return false } - ub := make([]float64, 0, len(p.tempNHCB.BucketCounts)) - for b := range p.tempNHCB.BucketCounts { - ub = append(ub, b) - } - upperBounds, hBase := convertnhcb.ProcessUpperBoundsAndCreateBaseHistogram(ub, false) - fhBase := hBase.ToFloat(nil) - h, fh := convertnhcb.NewHistogram(p.tempNHCB, upperBounds, hBase, fhBase) - if h != nil { - if err := h.Validate(); err != nil { - return false - } - p.hNHCB = h - p.fhNHCB = nil - } else if fh != nil { - if err := fh.Validate(); err != nil { - return false + h, fh, err := p.tempNHCB.Convert() + if err == nil { + if h != nil { + if err := h.Validate(); err != nil { + return false + } + p.hNHCB = h + p.fhNHCB = nil + } else if fh != nil { + if err := fh.Validate(); err != nil { + return false + } + p.hNHCB = nil + p.fhNHCB = fh } - p.hNHCB = nil - p.fhNHCB = fh + p.metricStringNHCB = p.tempLsetNHCB.Get(labels.MetricName) + strings.ReplaceAll(p.tempLsetNHCB.DropMetricName().String(), ", ", ",") + p.bytesNHCB = []byte(p.metricStringNHCB) + p.lsetNHCB = p.tempLsetNHCB + p.swapExemplars() + p.ctNHCB = p.tempCT + p.state = stateEmitting + } else { + p.state = stateStart } - p.metricStringNHCB = p.tempLsetNHCB.Get(labels.MetricName) + strings.ReplaceAll(p.tempLsetNHCB.DropMetricName().String(), ", ", ",") - p.bytesNHCB = []byte(p.metricStringNHCB) - p.lsetNHCB = p.tempLsetNHCB - p.swapExemplars() - p.ctNHCB = p.tempCT - p.tempNHCB = convertnhcb.NewTempHistogram() - p.state = stateEmitting + p.tempNHCB.Reset() + p.tempExemplarCount = 0 p.tempCT = nil - return true + return err == nil } diff --git a/vendor/github.com/prometheus/prometheus/promql/promqltest/test.go b/vendor/github.com/prometheus/prometheus/promql/promqltest/test.go index 9fc31ca0b80..4cbd39b403a 100644 --- a/vendor/github.com/prometheus/prometheus/promql/promqltest/test.go +++ b/vendor/github.com/prometheus/prometheus/promql/promqltest/test.go @@ -482,18 +482,16 @@ func (cmd *loadCmd) append(a storage.Appender) error { type tempHistogramWrapper struct { metric labels.Labels - upperBounds []float64 histogramByTs map[int64]convertnhcb.TempHistogram } func newTempHistogramWrapper() tempHistogramWrapper { return tempHistogramWrapper{ - upperBounds: []float64{}, histogramByTs: map[int64]convertnhcb.TempHistogram{}, } } -func processClassicHistogramSeries(m labels.Labels, suffix string, histogramMap map[uint64]tempHistogramWrapper, smpls []promql.Sample, updateHistogramWrapper func(*tempHistogramWrapper), updateHistogram func(*convertnhcb.TempHistogram, float64)) { +func processClassicHistogramSeries(m labels.Labels, suffix string, histogramMap map[uint64]tempHistogramWrapper, smpls []promql.Sample, updateHistogram func(*convertnhcb.TempHistogram, float64)) { m2 := convertnhcb.GetHistogramMetricBase(m, suffix) m2hash := m2.Hash() histogramWrapper, exists := histogramMap[m2hash] @@ -501,9 +499,6 @@ func processClassicHistogramSeries(m labels.Labels, suffix string, histogramMap histogramWrapper = newTempHistogramWrapper() } histogramWrapper.metric = m2 - if updateHistogramWrapper != nil { - updateHistogramWrapper(&histogramWrapper) - } for _, s := range smpls { if s.H != nil { continue @@ -534,18 +529,16 @@ func (cmd *loadCmd) appendCustomHistogram(a storage.Appender) error { if err != nil || math.IsNaN(le) { continue } - processClassicHistogramSeries(m, "_bucket", histogramMap, smpls, func(histogramWrapper *tempHistogramWrapper) { - histogramWrapper.upperBounds = append(histogramWrapper.upperBounds, le) - }, func(histogram *convertnhcb.TempHistogram, f float64) { - histogram.BucketCounts[le] = f + processClassicHistogramSeries(m, "_bucket", histogramMap, smpls, func(histogram *convertnhcb.TempHistogram, f float64) { + _ = histogram.SetBucketCount(le, f) }) case strings.HasSuffix(mName, "_count"): - processClassicHistogramSeries(m, "_count", histogramMap, smpls, nil, func(histogram *convertnhcb.TempHistogram, f float64) { - histogram.Count = f + processClassicHistogramSeries(m, "_count", histogramMap, smpls, func(histogram *convertnhcb.TempHistogram, f float64) { + _ = histogram.SetCount(f) }) case strings.HasSuffix(mName, "_sum"): - processClassicHistogramSeries(m, "_sum", histogramMap, smpls, nil, func(histogram *convertnhcb.TempHistogram, f float64) { - histogram.Sum = f + processClassicHistogramSeries(m, "_sum", histogramMap, smpls, func(histogram *convertnhcb.TempHistogram, f float64) { + _ = histogram.SetSum(f) }) } } @@ -553,11 +546,12 @@ func (cmd *loadCmd) appendCustomHistogram(a storage.Appender) error { // Convert the collated classic histogram data into native histograms // with custom bounds and append them to the storage. for _, histogramWrapper := range histogramMap { - upperBounds, hBase := convertnhcb.ProcessUpperBoundsAndCreateBaseHistogram(histogramWrapper.upperBounds, true) - fhBase := hBase.ToFloat(nil) samples := make([]promql.Sample, 0, len(histogramWrapper.histogramByTs)) for t, histogram := range histogramWrapper.histogramByTs { - h, fh := convertnhcb.NewHistogram(histogram, upperBounds, hBase, fhBase) + h, fh, err := histogram.Convert() + if err != nil { + return err + } if fh == nil { if err := h.Validate(); err != nil { return err diff --git a/vendor/github.com/prometheus/prometheus/scrape/manager.go b/vendor/github.com/prometheus/prometheus/scrape/manager.go index f3dad2a0488..04da3162e62 100644 --- a/vendor/github.com/prometheus/prometheus/scrape/manager.go +++ b/vendor/github.com/prometheus/prometheus/scrape/manager.go @@ -94,8 +94,6 @@ type Options struct { skipOffsetting bool } -const DefaultNameEscapingScheme = model.ValueEncodingEscaping - // Manager maintains a set of scrape pools and manages start/stop cycles // when receiving new target groups from the discovery manager. type Manager struct { diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/helpers_from_stdlib.go b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/helpers_from_stdlib.go new file mode 100644 index 00000000000..cb9257d0737 --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/helpers_from_stdlib.go @@ -0,0 +1,106 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// Provenance-includes-location: https://github.com/golang/go/blob/f2d118fd5f7e872804a5825ce29797f81a28b0fa/src/strings/strings.go +// Provenance-includes-license: BSD-3-Clause +// Provenance-includes-copyright: Copyright The Go Authors. + +package prometheus + +import "strings" + +// fieldsFunc is a copy of strings.FieldsFunc from the Go standard library, +// but it also returns the separators as part of the result. +func fieldsFunc(s string, f func(rune) bool) ([]string, []string) { + // A span is used to record a slice of s of the form s[start:end]. + // The start index is inclusive and the end index is exclusive. + type span struct { + start int + end int + } + spans := make([]span, 0, 32) + separators := make([]string, 0, 32) + + // Find the field start and end indices. + // Doing this in a separate pass (rather than slicing the string s + // and collecting the result substrings right away) is significantly + // more efficient, possibly due to cache effects. + start := -1 // valid span start if >= 0 + for end, rune := range s { + if f(rune) { + if start >= 0 { + spans = append(spans, span{start, end}) + // Set start to a negative value. + // Note: using -1 here consistently and reproducibly + // slows down this code by a several percent on amd64. + start = ^start + separators = append(separators, string(s[end])) + } + } else { + if start < 0 { + start = end + } + } + } + + // Last field might end at EOF. + if start >= 0 { + spans = append(spans, span{start, len(s)}) + } + + // Create strings from recorded field indices. + a := make([]string, len(spans)) + for i, span := range spans { + a[i] = s[span.start:span.end] + } + + return a, separators +} + +// join is a copy of strings.Join from the Go standard library, +// but it also accepts a slice of separators to join the elements with. +// If the slice of separators is shorter than the slice of elements, use a default value. +// We also don't check for integer overflow. +func join(elems []string, separators []string, def string) string { + switch len(elems) { + case 0: + return "" + case 1: + return elems[0] + } + + var n int + var sep string + sepLen := len(separators) + for i, elem := range elems { + if i >= sepLen { + sep = def + } else { + sep = separators[i] + } + n += len(sep) + len(elem) + } + + var b strings.Builder + b.Grow(n) + b.WriteString(elems[0]) + for i, s := range elems[1:] { + if i >= sepLen { + sep = def + } else { + sep = separators[i] + } + b.WriteString(sep) + b.WriteString(s) + } + return b.String() +} diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/normalize_label.go b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/normalize_label.go index d5de2c76518..b928e6888d1 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/normalize_label.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/normalize_label.go @@ -29,9 +29,9 @@ import ( // // Labels that start with non-letter rune will be prefixed with "key_". // An exception is made for double-underscores which are allowed. -func NormalizeLabel(label string) string { +func NormalizeLabel(label string, allowUTF8 bool) string { // Trivial case - if len(label) == 0 { + if len(label) == 0 || allowUTF8 { return label } diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/normalize_name.go b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/normalize_name.go index 0119b64dff0..335705aa8dd 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/normalize_name.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/normalize_name.go @@ -88,27 +88,32 @@ var perUnitMap = map[string]string{ // See rules at https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels, // https://prometheus.io/docs/practices/naming/#metric-and-label-naming // and https://github.com/open-telemetry/opentelemetry-specification/blob/v1.38.0/specification/compatibility/prometheus_and_openmetrics.md#otlp-metric-points-to-prometheus. -func BuildCompliantName(metric pmetric.Metric, namespace string, addMetricSuffixes bool) string { +func BuildCompliantName(metric pmetric.Metric, namespace string, addMetricSuffixes, allowUTF8 bool) string { // Full normalization following standard Prometheus naming conventions if addMetricSuffixes { - return normalizeName(metric, namespace) + return normalizeName(metric, namespace, allowUTF8) } - // Regexp for metric name characters that should be replaced with _. - invalidMetricCharRE := regexp.MustCompile(`[^a-zA-Z0-9:_]`) - - // Simple case (no full normalization, no units, etc.). - metricName := strings.Join(strings.FieldsFunc(metric.Name(), func(r rune) bool { - return invalidMetricCharRE.MatchString(string(r)) - }), "_") + var metricName string + if !allowUTF8 { + // Regexp for metric name characters that should be replaced with _. + invalidMetricCharRE := regexp.MustCompile(`[^a-zA-Z0-9:_]`) + + // Simple case (no full normalization, no units, etc.). + metricName = strings.Join(strings.FieldsFunc(metric.Name(), func(r rune) bool { + return invalidMetricCharRE.MatchString(string(r)) + }), "_") + } else { + metricName = metric.Name() + } // Namespace? if namespace != "" { return namespace + "_" + metricName } - // Metric name starts with a digit? Prefix it with an underscore. - if metricName != "" && unicode.IsDigit(rune(metricName[0])) { + // Metric name starts with a digit and utf8 not allowed? Prefix it with an underscore. + if metricName != "" && unicode.IsDigit(rune(metricName[0])) && !allowUTF8 { metricName = "_" + metricName } @@ -116,17 +121,18 @@ func BuildCompliantName(metric pmetric.Metric, namespace string, addMetricSuffix } // Build a normalized name for the specified metric. -func normalizeName(metric pmetric.Metric, namespace string) string { - // Regexp for characters that can't be in a metric name token. - nonTokenMetricCharRE := regexp.MustCompile(`[^a-zA-Z0-9:]`) - +func normalizeName(metric pmetric.Metric, namespace string, allowUTF8 bool) string { + var translationFunc func(rune) bool + if !allowUTF8 { + nonTokenMetricCharRE := regexp.MustCompile(`[^a-zA-Z0-9:]`) + translationFunc = func(r rune) bool { return nonTokenMetricCharRE.MatchString(string(r)) } + } else { + translationFunc = func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != ':' } + } // Split metric name into "tokens" (of supported metric name runes). // Note that this has the side effect of replacing multiple consecutive underscores with a single underscore. // This is part of the OTel to Prometheus specification: https://github.com/open-telemetry/opentelemetry-specification/blob/v1.38.0/specification/compatibility/prometheus_and_openmetrics.md#otlp-metric-points-to-prometheus. - nameTokens := strings.FieldsFunc( - metric.Name(), - func(r rune) bool { return nonTokenMetricCharRE.MatchString(string(r)) }, - ) + nameTokens, separators := fieldsFunc(metric.Name(), translationFunc) // Split unit at the '/' if any unitTokens := strings.SplitN(metric.Unit(), "/", 2) @@ -137,7 +143,10 @@ func normalizeName(metric pmetric.Metric, namespace string) string { var mainUnitProm, perUnitProm string mainUnitOTel := strings.TrimSpace(unitTokens[0]) if mainUnitOTel != "" && !strings.ContainsAny(mainUnitOTel, "{}") { - mainUnitProm = cleanUpUnit(unitMapGetOrDefault(mainUnitOTel)) + mainUnitProm = unitMapGetOrDefault(mainUnitOTel) + if !allowUTF8 { + mainUnitProm = cleanUpUnit(mainUnitProm) + } if slices.Contains(nameTokens, mainUnitProm) { mainUnitProm = "" } @@ -148,7 +157,10 @@ func normalizeName(metric pmetric.Metric, namespace string) string { if len(unitTokens) > 1 && unitTokens[1] != "" { perUnitOTel := strings.TrimSpace(unitTokens[1]) if perUnitOTel != "" && !strings.ContainsAny(perUnitOTel, "{}") { - perUnitProm = cleanUpUnit(perUnitMapGetOrDefault(perUnitOTel)) + perUnitProm = perUnitMapGetOrDefault(perUnitOTel) + if !allowUTF8 { + perUnitProm = cleanUpUnit(perUnitProm) + } } if perUnitProm != "" { perUnitProm = "per_" + perUnitProm @@ -189,8 +201,12 @@ func normalizeName(metric pmetric.Metric, namespace string) string { nameTokens = append([]string{namespace}, nameTokens...) } - // Build the string from the tokens, separated with underscores - normalizedName := strings.Join(nameTokens, "_") + // Build the string from the tokens + separators. + // If UTF-8 isn't allowed, we'll use underscores as separators. + if !allowUTF8 { + separators = []string{} + } + normalizedName := join(nameTokens, separators, "_") // Metric name cannot start with a digit, so prefix it with "_" in this case if normalizedName != "" && unicode.IsDigit(rune(normalizedName[0])) { diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/helper.go b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/helper.go index aefde1c2d7e..d08de7feced 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/helper.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/helper.go @@ -158,7 +158,7 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, setting // map ensures no duplicate label names. l := make(map[string]string, maxLabelCount) for _, label := range labels { - var finalKey = prometheustranslator.NormalizeLabel(label.Name) + var finalKey = prometheustranslator.NormalizeLabel(label.Name, settings.AllowUTF8) if existingValue, alreadyExists := l[finalKey]; alreadyExists { l[finalKey] = existingValue + ";" + label.Value } else { @@ -167,7 +167,7 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, setting } for _, lbl := range promotedAttrs { - normalized := prometheustranslator.NormalizeLabel(lbl.Name) + normalized := prometheustranslator.NormalizeLabel(lbl.Name, settings.AllowUTF8) if _, exists := l[normalized]; !exists { l[normalized] = lbl.Value } @@ -206,7 +206,7 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, setting } // internal labels should be maintained if !(len(name) > 4 && name[:2] == "__" && name[len(name)-2:] == "__") { - name = prometheustranslator.NormalizeLabel(name) + name = prometheustranslator.NormalizeLabel(name, settings.AllowUTF8) } l[name] = extras[i+1] } diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go index a7f41a63164..3dde6ce6a17 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go @@ -35,13 +35,14 @@ import ( ) type Settings struct { - ExternalLabels map[string]string Namespace string - PromoteResourceAttributes []string + ExternalLabels map[string]string DisableTargetInfo bool ExportCreatedMetric bool AddMetricSuffixes bool SendMetadata bool + AllowUTF8 bool + PromoteResourceAttributes []string EnableCreatedTimestampZeroIngestion bool ValidIntervalCreatedTimestampZeroIngestion time.Duration } @@ -95,7 +96,7 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric continue } - promName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) + promName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes, settings.AllowUTF8) // handle individual metrics based on type //exhaustive:enforce diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/otlp_to_openmetrics_metadata.go b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/otlp_to_openmetrics_metadata.go index ba487041930..b423d2cc6e4 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/otlp_to_openmetrics_metadata.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/otlp_to_openmetrics_metadata.go @@ -43,7 +43,7 @@ func otelMetricTypeToPromMetricType(otelMetric pmetric.Metric) prompb.MetricMeta return prompb.MetricMetadata_UNKNOWN } -func OtelMetricsToMetadata(md pmetric.Metrics, addMetricSuffixes bool) []*prompb.MetricMetadata { +func OtelMetricsToMetadata(md pmetric.Metrics, addMetricSuffixes, allowUTF8 bool) []*prompb.MetricMetadata { resourceMetricsSlice := md.ResourceMetrics() metadataLength := 0 @@ -65,7 +65,7 @@ func OtelMetricsToMetadata(md pmetric.Metrics, addMetricSuffixes bool) []*prompb metric := scopeMetrics.Metrics().At(k) entry := prompb.MetricMetadata{ Type: otelMetricTypeToPromMetricType(metric), - MetricFamilyName: prometheustranslator.BuildCompliantName(metric, "", addMetricSuffixes), + MetricFamilyName: prometheustranslator.BuildCompliantName(metric, "", addMetricSuffixes, allowUTF8), Help: metric.Description(), } metadata = append(metadata, &entry) diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/write_handler.go b/vendor/github.com/prometheus/prometheus/storage/remote/write_handler.go index 99e4392ff5e..e15c01bed7a 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/write_handler.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/write_handler.go @@ -517,6 +517,7 @@ func (h *otlpWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { converter := otlptranslator.NewPrometheusConverter() annots, err := converter.FromMetrics(r.Context(), req.Metrics(), otlptranslator.Settings{ AddMetricSuffixes: true, + AllowUTF8: otlpCfg.TranslationStrategy == config.NoUTF8EscapingWithSuffixes, PromoteResourceAttributes: otlpCfg.PromoteResourceAttributes, EnableCreatedTimestampZeroIngestion: h.enableCTZeroIngestion, ValidIntervalCreatedTimestampZeroIngestion: h.validIntervalCTZeroIngestion, diff --git a/vendor/github.com/prometheus/prometheus/tsdb/head_append.go b/vendor/github.com/prometheus/prometheus/tsdb/head_append.go index 7d0122e0b96..e16bd73b295 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/head_append.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/head_append.go @@ -356,21 +356,21 @@ func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64 } } + s.Lock() if value.IsStaleNaN(v) { - // This is not thread safe as we should be holding the lock for "s". // TODO(krajorama): reorganize Commit() to handle samples in append order // not floats first and then histograms. Then we could do this conversion // in commit. This code should move into Commit(). switch { case s.lastHistogramValue != nil: + s.Unlock() return a.AppendHistogram(ref, lset, t, &histogram.Histogram{Sum: v}, nil) case s.lastFloatHistogramValue != nil: + s.Unlock() return a.AppendHistogram(ref, lset, t, nil, &histogram.FloatHistogram{Sum: v}) } } - s.Lock() - defer s.Unlock() // TODO(codesome): If we definitely know at this point that the sample is ooo, then optimise // to skip that sample from the WAL and write only in the WBL. @@ -505,6 +505,10 @@ func (s *memSeries) appendable(t int64, v float64, headMaxt, minValidTime, oooTi } } + if math.Float64bits(v) == value.QuietZeroNaN { // Say it's allowed; it will be dropped later in commitSamples. + return true, 0, nil + } + // The sample cannot go in the in-order chunk. Check if it can go in the out-of-order chunk. if oooTimeWindow > 0 && t >= headMaxt-oooTimeWindow { return true, headMaxt - t, nil @@ -1141,13 +1145,11 @@ func (a *headAppender) commitSamples(acc *appenderCommitContext) { handleAppendableError(err, &acc.floatsAppended, &acc.floatOOORejected, &acc.floatOOBRejected, &acc.floatTooOldRejected) } - if math.Float64bits(s.V) == value.QuietZeroNaN { - s.V = 0 - } - switch { case err != nil: // Do nothing here. + case oooSample && math.Float64bits(s.V) == value.QuietZeroNaN: + // No-op: we don't store quiet zeros out-of-order. case oooSample: // Sample is OOO and OOO handling is enabled // and the delta is within the OOO tolerance. @@ -1194,6 +1196,9 @@ func (a *headAppender) commitSamples(acc *appenderCommitContext) { acc.floatsAppended-- } default: + if math.Float64bits(s.V) == value.QuietZeroNaN { + s.V = 0 + } ok, chunkCreated = series.append(s.T, s.V, a.appendID, acc.appendChunkOpts) if ok { if s.T < acc.inOrderMint { @@ -1521,7 +1526,7 @@ type chunkOpts struct { // append adds the sample (t, v) to the series. The caller also has to provide // the appendID for isolation. (The appendID can be zero, which results in no // isolation for this append.) -// It is unsafe to call this concurrently with s.iterator(...) without holding the series lock. +// Series lock must be held when calling. func (s *memSeries) append(t int64, v float64, appendID uint64, o chunkOpts) (sampleInOrder, chunkCreated bool) { c, sampleInOrder, chunkCreated := s.appendPreprocessor(t, chunkenc.EncXOR, o) if !sampleInOrder { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/querier.go b/vendor/github.com/prometheus/prometheus/tsdb/querier.go index 44a542e3766..36b22567911 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/querier.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/querier.go @@ -192,6 +192,11 @@ func selectChunkSeriesSet(ctx context.Context, sortSeries bool, hints *storage.S // PostingsForMatchers assembles a single postings iterator against the index reader // based on the given matchers. The resulting postings are not ordered by series. func PostingsForMatchers(ctx context.Context, ix IndexPostingsReader, ms ...*labels.Matcher) (index.Postings, error) { + if len(ms) == 1 && ms[0].Name == "" && ms[0].Value == "" { + k, v := index.AllPostingsKey() + return ix.Postings(ctx, k, v) + } + var its, notIts []index.Postings // See which label must be non-empty. // Optimization for case like {l=~".", l!="1"}. @@ -246,13 +251,10 @@ func PostingsForMatchers(ctx context.Context, ix IndexPostingsReader, ms ...*lab return nil, ctx.Err() } switch { - case m.Name == "" && m.Value == "": // Special-case for AllPostings, used in tests at least. - k, v := index.AllPostingsKey() - allPostings, err := ix.Postings(ctx, k, v) - if err != nil { - return nil, err - } - its = append(its, allPostings) + case m.Name == "" && m.Value == "": + // We already handled the case at the top of the function, + // and it is unexpected to get all postings again here. + return nil, errors.New("unexpected all postings") case m.Type == labels.MatchRegexp && m.Value == ".*": // .* regexp matches any string: do nothing. case m.Type == labels.MatchNotRegexp && m.Value == ".*": diff --git a/vendor/github.com/prometheus/prometheus/util/convertnhcb/convertnhcb.go b/vendor/github.com/prometheus/prometheus/util/convertnhcb/convertnhcb.go index 5e08422aa06..ee5bcb72def 100644 --- a/vendor/github.com/prometheus/prometheus/util/convertnhcb/convertnhcb.go +++ b/vendor/github.com/prometheus/prometheus/util/convertnhcb/convertnhcb.go @@ -14,6 +14,7 @@ package convertnhcb import ( + "errors" "fmt" "math" "sort" @@ -23,129 +24,212 @@ import ( "github.com/prometheus/prometheus/model/labels" ) +var ( + errNegativeBucketCount = errors.New("bucket count must be non-negative") + errNegativeCount = errors.New("count must be non-negative") + errCountMismatch = errors.New("count mismatch") + errCountNotCumulative = errors.New("count is not cumulative") +) + +type tempHistogramBucket struct { + le float64 + count float64 +} + // TempHistogram is used to collect information about classic histogram // samples incrementally before creating a histogram.Histogram or // histogram.FloatHistogram based on the values collected. type TempHistogram struct { - BucketCounts map[float64]float64 - Count float64 - Sum float64 - HasFloat bool + buckets []tempHistogramBucket + count float64 + sum float64 + err error + hasCount bool } // NewTempHistogram creates a new TempHistogram to // collect information about classic histogram samples. func NewTempHistogram() TempHistogram { return TempHistogram{ - BucketCounts: map[float64]float64{}, + buckets: make([]tempHistogramBucket, 0, 10), } } -func (h TempHistogram) getIntBucketCounts() (map[float64]int64, error) { - bucketCounts := map[float64]int64{} - for le, count := range h.BucketCounts { - intCount := int64(math.Round(count)) - if float64(intCount) != count { - return nil, fmt.Errorf("bucket count %f for le %g is not an integer", count, le) +func (h TempHistogram) Err() error { + return h.err +} + +func (h *TempHistogram) Reset() { + h.buckets = h.buckets[:0] + h.count = 0 + h.sum = 0 + h.err = nil + h.hasCount = false +} + +func (h *TempHistogram) SetBucketCount(boundary, count float64) error { + if h.err != nil { + return h.err + } + if count < 0 { + h.err = fmt.Errorf("%w: le=%g, count=%g", errNegativeBucketCount, boundary, count) + return h.err + } + // Assume that the elements are added in order. + switch { + case len(h.buckets) == 0: + h.buckets = append(h.buckets, tempHistogramBucket{le: boundary, count: count}) + case h.buckets[len(h.buckets)-1].le < boundary: + // Happy case is "<". + if count < h.buckets[len(h.buckets)-1].count { + h.err = fmt.Errorf("%w: %g < %g", errCountNotCumulative, count, h.buckets[len(h.buckets)-1].count) + return h.err } - bucketCounts[le] = intCount + h.buckets = append(h.buckets, tempHistogramBucket{le: boundary, count: count}) + case h.buckets[len(h.buckets)-1].le == boundary: + // Ignore this, as it is a duplicate sample. + default: + // Find the correct position to insert. + i := sort.Search(len(h.buckets), func(i int) bool { + return h.buckets[i].le >= boundary + }) + if h.buckets[i].le == boundary { + // Ignore this, as it is a duplicate sample. + return nil + } + if i > 0 && count < h.buckets[i-1].count { + h.err = fmt.Errorf("%w: %g < %g", errCountNotCumulative, count, h.buckets[i-1].count) + return h.err + } + if count > h.buckets[i].count { + h.err = fmt.Errorf("%w: %g > %g", errCountNotCumulative, count, h.buckets[i].count) + return h.err + } + // Insert at the correct position unless duplicate. + h.buckets = append(h.buckets, tempHistogramBucket{}) + copy(h.buckets[i+1:], h.buckets[i:]) + h.buckets[i] = tempHistogramBucket{le: boundary, count: count} } - return bucketCounts, nil + return nil } -// ProcessUpperBoundsAndCreateBaseHistogram prepares an integer native -// histogram with custom buckets based on the provided upper bounds. -// Everything is set except the bucket counts. -// The sorted upper bounds are also returned. -func ProcessUpperBoundsAndCreateBaseHistogram(upperBounds0 []float64, needsDedup bool) ([]float64, *histogram.Histogram) { - sort.Float64s(upperBounds0) - var upperBounds []float64 - if needsDedup { - upperBounds = make([]float64, 0, len(upperBounds0)) - prevLE := math.Inf(-1) - for _, le := range upperBounds0 { - if le != prevLE { - upperBounds = append(upperBounds, le) - prevLE = le - } - } - } else { - upperBounds = upperBounds0 - } - var customBounds []float64 - if upperBounds[len(upperBounds)-1] == math.Inf(1) { - customBounds = upperBounds[:len(upperBounds)-1] - } else { - customBounds = upperBounds - } - return upperBounds, &histogram.Histogram{ - Count: 0, - Sum: 0, - Schema: histogram.CustomBucketsSchema, - PositiveSpans: []histogram.Span{ - {Offset: 0, Length: uint32(len(upperBounds))}, - }, - PositiveBuckets: make([]int64, len(upperBounds)), - CustomValues: customBounds, +func (h *TempHistogram) SetCount(count float64) error { + if h.err != nil { + return h.err + } + if count < 0 { + h.err = fmt.Errorf("%w: count=%g", errNegativeCount, count) + return h.err } + h.count = count + h.hasCount = true + return nil } -// NewHistogram fills the bucket counts in the provided histogram.Histogram -// or histogram.FloatHistogram based on the provided temporary histogram and -// upper bounds. -func NewHistogram(histogram TempHistogram, upperBounds []float64, hBase *histogram.Histogram, fhBase *histogram.FloatHistogram) (*histogram.Histogram, *histogram.FloatHistogram) { - intBucketCounts, err := histogram.getIntBucketCounts() - if err != nil { - return nil, newFloatHistogram(histogram, upperBounds, histogram.BucketCounts, fhBase) +func (h *TempHistogram) SetSum(sum float64) error { + if h.err != nil { + return h.err } - return newIntegerHistogram(histogram, upperBounds, intBucketCounts, hBase), nil + h.sum = sum + return nil } -func newIntegerHistogram(histogram TempHistogram, upperBounds []float64, bucketCounts map[float64]int64, hBase *histogram.Histogram) *histogram.Histogram { - h := hBase.Copy() - absBucketCounts := make([]int64, len(h.PositiveBuckets)) - var prevCount, total int64 - for i, le := range upperBounds { - currCount, exists := bucketCounts[le] - if !exists { - currCount = 0 +func (h TempHistogram) Convert() (*histogram.Histogram, *histogram.FloatHistogram, error) { + if h.err != nil { + return nil, nil, h.err + } + + if len(h.buckets) == 0 || h.buckets[len(h.buckets)-1].le != math.Inf(1) { + // No +Inf bucket. + if !h.hasCount && len(h.buckets) > 0 { + // No count either, so set count to the last known bucket's count. + h.count = h.buckets[len(h.buckets)-1].count } - count := currCount - prevCount - absBucketCounts[i] = count - total += count - prevCount = currCount + // Let the last bucket be +Inf with the overall count. + h.buckets = append(h.buckets, tempHistogramBucket{le: math.Inf(1), count: h.count}) } - h.PositiveBuckets[0] = absBucketCounts[0] - for i := 1; i < len(h.PositiveBuckets); i++ { - h.PositiveBuckets[i] = absBucketCounts[i] - absBucketCounts[i-1] + + if !h.hasCount { + h.count = h.buckets[len(h.buckets)-1].count + h.hasCount = true } - h.Sum = histogram.Sum - if histogram.Count != 0 { - total = int64(histogram.Count) + + for _, b := range h.buckets { + intCount := int64(math.Round(b.count)) + if b.count != float64(intCount) { + return h.convertToFloatHistogram() + } } - h.Count = uint64(total) - return h.Compact(0) + + intCount := uint64(math.Round(h.count)) + if h.count != float64(intCount) { + return h.convertToFloatHistogram() + } + return h.convertToIntegerHistogram(intCount) } -func newFloatHistogram(histogram TempHistogram, upperBounds []float64, bucketCounts map[float64]float64, fhBase *histogram.FloatHistogram) *histogram.FloatHistogram { - fh := fhBase.Copy() - var prevCount, total float64 - for i, le := range upperBounds { - currCount, exists := bucketCounts[le] - if !exists { - currCount = 0 +func (h TempHistogram) convertToIntegerHistogram(count uint64) (*histogram.Histogram, *histogram.FloatHistogram, error) { + rh := &histogram.Histogram{ + Schema: histogram.CustomBucketsSchema, + Count: count, + Sum: h.sum, + PositiveSpans: []histogram.Span{{Length: uint32(len(h.buckets))}}, + PositiveBuckets: make([]int64, len(h.buckets)), + } + + if len(h.buckets) > 1 { + rh.CustomValues = make([]float64, len(h.buckets)-1) // Not storing the last +Inf bucket. + } + + prevCount := int64(0) + prevDelta := int64(0) + for i, b := range h.buckets { + // delta is the actual bucket count as the input is cumulative. + delta := int64(b.count) - prevCount + rh.PositiveBuckets[i] = delta - prevDelta + prevCount = int64(b.count) + prevDelta = delta + if b.le != math.Inf(1) { + rh.CustomValues[i] = b.le } - count := currCount - prevCount - fh.PositiveBuckets[i] = count - total += count - prevCount = currCount } - fh.Sum = histogram.Sum - if histogram.Count != 0 { - total = histogram.Count + + if count != uint64(h.buckets[len(h.buckets)-1].count) { + h.err = fmt.Errorf("%w: count=%d != le=%g count=%g", errCountMismatch, count, h.buckets[len(h.buckets)-1].le, h.buckets[len(h.buckets)-1].count) + return nil, nil, h.err } - fh.Count = total - return fh.Compact(0) + + return rh.Compact(2), nil, nil +} + +func (h TempHistogram) convertToFloatHistogram() (*histogram.Histogram, *histogram.FloatHistogram, error) { + rh := &histogram.FloatHistogram{ + Schema: histogram.CustomBucketsSchema, + Count: h.count, + Sum: h.sum, + PositiveSpans: []histogram.Span{{Length: uint32(len(h.buckets))}}, + PositiveBuckets: make([]float64, len(h.buckets)), + } + + if len(h.buckets) > 1 { + rh.CustomValues = make([]float64, len(h.buckets)-1) // Not storing the last +Inf bucket. + } + + prevCount := 0.0 + for i, b := range h.buckets { + rh.PositiveBuckets[i] = b.count - prevCount + prevCount = b.count + if b.le != math.Inf(1) { + rh.CustomValues[i] = b.le + } + } + + if h.count != h.buckets[len(h.buckets)-1].count { + h.err = fmt.Errorf("%w: count=%g != le=%g count=%g", errCountMismatch, h.count, h.buckets[len(h.buckets)-1].le, h.buckets[len(h.buckets)-1].count) + return nil, nil, h.err + } + + return nil, rh.Compact(0), nil } func GetHistogramMetricBase(m labels.Labels, suffix string) labels.Labels { diff --git a/vendor/modules.txt b/vendor/modules.txt index 6eeb8806898..432119f01b9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1009,7 +1009,7 @@ github.com/prometheus/common/version # github.com/prometheus/common/sigv4 v0.1.0 ## explicit; go 1.15 github.com/prometheus/common/sigv4 -# github.com/prometheus/exporter-toolkit v0.13.0 +# github.com/prometheus/exporter-toolkit v0.13.1 ## explicit; go 1.22 github.com/prometheus/exporter-toolkit/web # github.com/prometheus/procfs v0.15.1 @@ -1017,7 +1017,7 @@ github.com/prometheus/exporter-toolkit/web github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util -# github.com/prometheus/prometheus v1.99.0 => github.com/grafana/mimir-prometheus v0.0.0-20241202100552-120c71abae28 +# github.com/prometheus/prometheus v1.99.0 => github.com/grafana/mimir-prometheus v0.0.0-20241203165959-320e9aa3985d ## explicit; go 1.22.0 github.com/prometheus/prometheus/config github.com/prometheus/prometheus/discovery @@ -1685,7 +1685,7 @@ sigs.k8s.io/kustomize/kyaml/yaml/walk sigs.k8s.io/yaml sigs.k8s.io/yaml/goyaml.v2 sigs.k8s.io/yaml/goyaml.v3 -# github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20241202100552-120c71abae28 +# github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20241203165959-320e9aa3985d # github.com/hashicorp/memberlist => github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe # gopkg.in/yaml.v3 => github.com/colega/go-yaml-yaml v0.0.0-20220720105220-255a8d16d094 # github.com/grafana/regexp => github.com/grafana/regexp v0.0.0-20240531075221-3685f1377d7b