From 539e690c75a569f66a0af0630922ddcc4eb7544c Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 9 Oct 2024 14:05:28 -0400 Subject: [PATCH] query-tee: Add an option to skip samples for comparison before a given timestamp (#9515) * query-tee: Add an option to skip samples for comparison before a given timestamp Signed-off-by: Ganesh Vernekar * CHANGELOG Signed-off-by: Ganesh Vernekar * Change flag type Signed-off-by: Ganesh Vernekar * Fix review comments Signed-off-by: Ganesh Vernekar * update tests with the new error message Signed-off-by: Ganesh Vernekar * make linter happy Signed-off-by: Ganesh Vernekar * Fix review comments Signed-off-by: Ganesh Vernekar * Update flag description Signed-off-by: Ganesh Vernekar * Update tools/querytee/response_comparator.go Co-authored-by: Marco Pracucci * Update tools/querytee/response_comparator.go Co-authored-by: Marco Pracucci * Fix review comments Signed-off-by: Ganesh Vernekar --------- Signed-off-by: Ganesh Vernekar Co-authored-by: Marco Pracucci --- CHANGELOG.md | 2 + cmd/query-tee/main.go | 3 + tools/querytee/proxy.go | 3 + tools/querytee/response_comparator.go | 105 +++++++-- tools/querytee/response_comparator_test.go | 248 +++++++++++++++++++++ 5 files changed, 346 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5517d3960e..721932e6b7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,8 @@ ### Query-tee +* [FEATURE] Added `-proxy.compare-skip-samples-before` to skip samples before the given time when comparing responses. The time can be in RFC3339 format (or) RFC3339 without the timezone and seconds (or) date only. #9515 + ### Documentation ### Tools diff --git a/cmd/query-tee/main.go b/cmd/query-tee/main.go index 7ebb2089fb7..d3ab70fcca7 100644 --- a/cmd/query-tee/main.go +++ b/cmd/query-tee/main.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "os" + "time" "github.com/go-kit/log/level" "github.com/grafana/dskit/flagext" @@ -17,6 +18,7 @@ import ( "github.com/grafana/dskit/tracing" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/common/model" jaegercfg "github.com/uber/jaeger-client-go/config" "github.com/grafana/mimir/pkg/util/instrumentation" @@ -106,6 +108,7 @@ func mimirReadRoutes(cfg Config) []querytee.Route { Tolerance: cfg.ProxyConfig.ValueComparisonTolerance, UseRelativeError: cfg.ProxyConfig.UseRelativeError, SkipRecentSamples: cfg.ProxyConfig.SkipRecentSamples, + SkipSamplesBefore: model.Time(time.Time(cfg.ProxyConfig.SkipSamplesBefore).UnixMilli()), RequireExactErrorMatch: cfg.ProxyConfig.RequireExactErrorMatch, }) diff --git a/tools/querytee/proxy.go b/tools/querytee/proxy.go index 02d1e9f43a8..3a4f52e21bb 100644 --- a/tools/querytee/proxy.go +++ b/tools/querytee/proxy.go @@ -18,6 +18,7 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" + "github.com/grafana/dskit/flagext" "github.com/grafana/dskit/server" "github.com/grafana/dskit/spanlogger" "github.com/pkg/errors" @@ -39,6 +40,7 @@ type ProxyConfig struct { UseRelativeError bool PassThroughNonRegisteredRoutes bool SkipRecentSamples time.Duration + SkipSamplesBefore flagext.Time RequireExactErrorMatch bool BackendSkipTLSVerify bool AddMissingTimeParamToInstantQueries bool @@ -65,6 +67,7 @@ func (cfg *ProxyConfig) RegisterFlags(f *flag.FlagSet) { f.Float64Var(&cfg.ValueComparisonTolerance, "proxy.value-comparison-tolerance", 0.000001, "The tolerance to apply when comparing floating point values in the responses. 0 to disable tolerance and require exact match (not recommended).") f.BoolVar(&cfg.UseRelativeError, "proxy.compare-use-relative-error", false, "Use relative error tolerance when comparing floating point values.") f.DurationVar(&cfg.SkipRecentSamples, "proxy.compare-skip-recent-samples", 2*time.Minute, "The window from now to skip comparing samples. 0 to disable.") + f.Var(&cfg.SkipSamplesBefore, "proxy.compare-skip-samples-before", "Skip the samples before the given time for comparison. The time can be in RFC3339 format (or) RFC3339 without the timezone and seconds (or) date only.") f.BoolVar(&cfg.RequireExactErrorMatch, "proxy.compare-exact-error-matching", false, "If true, errors will be considered the same only if they are exactly the same. If false, errors will be considered the same if they are considered equivalent.") f.BoolVar(&cfg.PassThroughNonRegisteredRoutes, "proxy.passthrough-non-registered-routes", false, "Passthrough requests for non-registered routes to preferred backend.") f.BoolVar(&cfg.AddMissingTimeParamToInstantQueries, "proxy.add-missing-time-parameter-to-instant-queries", true, "Add a 'time' parameter to proxied instant query requests if they do not have one.") diff --git a/tools/querytee/response_comparator.go b/tools/querytee/response_comparator.go index f992a279fe8..69e015f8dd0 100644 --- a/tools/querytee/response_comparator.go +++ b/tools/querytee/response_comparator.go @@ -40,6 +40,7 @@ type SampleComparisonOptions struct { Tolerance float64 UseRelativeError bool SkipRecentSamples time.Duration + SkipSamplesBefore model.Time RequireExactErrorMatch bool } @@ -220,7 +221,7 @@ func compareMatrix(expectedRaw, actualRaw json.RawMessage, queryEvaluationTime t return err } - if allMatrixSamplesWithinRecentSampleWindow(expected, queryEvaluationTime, opts) && allMatrixSamplesWithinRecentSampleWindow(actual, queryEvaluationTime, opts) { + if allMatrixSamplesOutsideComparableWindow(expected, queryEvaluationTime, opts) && allMatrixSamplesOutsideComparableWindow(actual, queryEvaluationTime, opts) { return nil } @@ -250,6 +251,19 @@ func compareMatrix(expectedRaw, actualRaw json.RawMessage, queryEvaluationTime t } func compareMatrixSamples(expected, actual *model.SampleStream, queryEvaluationTime time.Time, opts SampleComparisonOptions) error { + expected.Values = trimBeginning(expected.Values, func(p model.SamplePair) bool { + return p.Timestamp < opts.SkipSamplesBefore + }) + actual.Values = trimBeginning(actual.Values, func(p model.SamplePair) bool { + return p.Timestamp < opts.SkipSamplesBefore + }) + expected.Histograms = trimBeginning(expected.Histograms, func(p model.SampleHistogramPair) bool { + return p.Timestamp < opts.SkipSamplesBefore + }) + actual.Histograms = trimBeginning(actual.Histograms, func(p model.SampleHistogramPair) bool { + return p.Timestamp < opts.SkipSamplesBefore + }) + expectedSamplesTail, actualSamplesTail, err := comparePairs(expected.Values, actual.Values, func(p1 model.SamplePair, p2 model.SamplePair) error { err := compareSamplePair(p1, p2, queryEvaluationTime, opts) if err != nil { @@ -281,15 +295,15 @@ func compareMatrixSamples(expected, actual *model.SampleStream, queryEvaluationT return nil } - skipAllRecentFloatSamples := canSkipAllSamples(func(p model.SamplePair) bool { - return queryEvaluationTime.Sub(p.Timestamp.Time())-opts.SkipRecentSamples < 0 + skipAllFloatSamples := canSkipAllSamples(func(p model.SamplePair) bool { + return queryEvaluationTime.Sub(p.Timestamp.Time())-opts.SkipRecentSamples < 0 || p.Timestamp < opts.SkipSamplesBefore }, expectedSamplesTail, actualSamplesTail) - skipAllRecentHistogramSamples := canSkipAllSamples(func(p model.SampleHistogramPair) bool { - return queryEvaluationTime.Sub(p.Timestamp.Time())-opts.SkipRecentSamples < 0 + skipAllHistogramSamples := canSkipAllSamples(func(p model.SampleHistogramPair) bool { + return queryEvaluationTime.Sub(p.Timestamp.Time())-opts.SkipRecentSamples < 0 || p.Timestamp < opts.SkipSamplesBefore }, expectedHistogramSamplesTail, actualHistogramSamplesTail) - if skipAllRecentFloatSamples && skipAllRecentHistogramSamples { + if skipAllFloatSamples && skipAllHistogramSamples { return nil } @@ -345,6 +359,28 @@ func comparePairs[S ~[]M, M any](s1, s2 S, compare func(M, M) error) (S, S, erro return s1[i:], s2[i:], nil } +// trimBeginning returns s without the prefix that satisfies skip(). +func trimBeginning[S ~[]M, M any](s S, skip func(M) bool) S { + var i int + for ; i < len(s); i++ { + if !skip(s[i]) { + break + } + } + return s[i:] +} + +// filterSlice returns a new slice with elements from s removed that satisfy skip(). +func filterSlice[S ~[]M, M any](s S, skip func(M) bool) S { + res := make(S, 0, len(s)) + for i := 0; i < len(s); i++ { + if !skip(s[i]) { + res = append(res, s[i]) + } + } + return res +} + func canSkipAllSamples[S ~[]M, M any](skip func(M) bool, ss ...S) bool { for _, s := range ss { for _, p := range s { @@ -356,20 +392,22 @@ func canSkipAllSamples[S ~[]M, M any](skip func(M) bool, ss ...S) bool { return true } -func allMatrixSamplesWithinRecentSampleWindow(m model.Matrix, queryEvaluationTime time.Time, opts SampleComparisonOptions) bool { - if opts.SkipRecentSamples == 0 { +func allMatrixSamplesOutsideComparableWindow(m model.Matrix, queryEvaluationTime time.Time, opts SampleComparisonOptions) bool { + if opts.SkipRecentSamples == 0 && opts.SkipSamplesBefore == 0 { return false } for _, series := range m { for _, sample := range series.Values { - if queryEvaluationTime.Sub(sample.Timestamp.Time()) > opts.SkipRecentSamples { + st := sample.Timestamp + if queryEvaluationTime.Sub(st.Time()) > opts.SkipRecentSamples && st >= opts.SkipSamplesBefore { return false } } for _, sample := range series.Histograms { - if queryEvaluationTime.Sub(sample.Timestamp.Time()) > opts.SkipRecentSamples { + st := sample.Timestamp + if queryEvaluationTime.Sub(st.Time()) > opts.SkipRecentSamples && st >= opts.SkipSamplesBefore { return false } } @@ -378,7 +416,7 @@ func allMatrixSamplesWithinRecentSampleWindow(m model.Matrix, queryEvaluationTim return true } -func compareVector(expectedRaw, actualRaw json.RawMessage, queryEvaluationTime time.Time, opts SampleComparisonOptions) error { +func compareVector(expectedRaw, actualRaw json.RawMessage, queryEvaluationTime time.Time, opts SampleComparisonOptions) (retErr error) { var expected, actual model.Vector err := json.Unmarshal(expectedRaw, &expected) @@ -391,10 +429,36 @@ func compareVector(expectedRaw, actualRaw json.RawMessage, queryEvaluationTime t return err } - if allVectorSamplesWithinRecentSampleWindow(expected, queryEvaluationTime, opts) && allVectorSamplesWithinRecentSampleWindow(actual, queryEvaluationTime, opts) { + if allVectorSamplesOutsideComparableWindow(expected, queryEvaluationTime, opts) && allVectorSamplesOutsideComparableWindow(actual, queryEvaluationTime, opts) { return nil } + if opts.SkipSamplesBefore > 0 { + // Warning: filtering samples can give out confusing error messages. For example if the actual response had + // matching series, but all sample timestamps were before SkipSamplesBefore, while expected response had samples + // after SkipSamplesBefore: instead of saying mismatch, it will instead say that some metrics is missing, because + // we filter them here. + eOrgLen, aOrgLen := len(expected), len(actual) + expected = filterSlice(expected, func(p *model.Sample) bool { return p.Timestamp < opts.SkipSamplesBefore }) + actual = filterSlice(actual, func(p *model.Sample) bool { return p.Timestamp < opts.SkipSamplesBefore }) + eChanged, aChanged := len(expected) != eOrgLen, len(actual) != aOrgLen + defer func() { + if retErr != nil { + warning := "" + if eChanged && aChanged { + warning = " (also, some samples were filtered out from the expected and actual response due to the 'skip samples before'; if all samples have been filtered out, this could cause the check on the expected number of metrics to fail)" + } else if aChanged { + warning = " (also, some samples were filtered out from the actual response due to the 'skip samples before'; if all samples have been filtered out, this could cause the check on the expected number of metrics to fail)" + } else if eChanged { + warning = " (also, some samples were filtered out from the expected response due to the 'skip samples before'; if all samples have been filtered out, this could cause the check on the expected number of metrics to fail)" + } + if warning != "" { + retErr = fmt.Errorf("%w%s", retErr, warning) + } + } + }() + } + if len(expected) != len(actual) { return fmt.Errorf("expected %d metrics but got %d", len(expected), len(actual)) } @@ -454,13 +518,14 @@ func compareVector(expectedRaw, actualRaw json.RawMessage, queryEvaluationTime t return nil } -func allVectorSamplesWithinRecentSampleWindow(v model.Vector, queryEvaluationTime time.Time, opts SampleComparisonOptions) bool { - if opts.SkipRecentSamples == 0 { +func allVectorSamplesOutsideComparableWindow(v model.Vector, queryEvaluationTime time.Time, opts SampleComparisonOptions) bool { + if opts.SkipRecentSamples == 0 && opts.SkipSamplesBefore == 0 { return false } for _, sample := range v { - if queryEvaluationTime.Sub(sample.Timestamp.Time()) > opts.SkipRecentSamples { + st := sample.Timestamp + if queryEvaluationTime.Sub(st.Time()) > opts.SkipRecentSamples && st >= opts.SkipSamplesBefore { return false } } @@ -495,6 +560,12 @@ func compareScalar(expectedRaw, actualRaw json.RawMessage, queryEvaluationTime t } func compareSamplePair(expected, actual model.SamplePair, queryEvaluationTime time.Time, opts SampleComparisonOptions) error { + // If the timestamp is before the configured SkipSamplesBefore then we don't even check if the timestamp is correct. + // The reason is that the SkipSamplesBefore feature may be used to compare queries hitting a different storage and one of two storages has no historical data. + if expected.Timestamp < opts.SkipSamplesBefore && actual.Timestamp < opts.SkipSamplesBefore { + return nil + } + if expected.Timestamp != actual.Timestamp { return fmt.Errorf("expected timestamp %v but got %v", expected.Timestamp, actual.Timestamp) } @@ -523,6 +594,10 @@ func compareSampleValue(first, second float64, opts SampleComparisonOptions) boo } func compareSampleHistogramPair(expected, actual model.SampleHistogramPair, queryEvaluationTime time.Time, opts SampleComparisonOptions) error { + if expected.Timestamp < opts.SkipSamplesBefore && actual.Timestamp < opts.SkipSamplesBefore { + return nil + } + if expected.Timestamp != actual.Timestamp { return fmt.Errorf("expected timestamp %v but got %v", expected.Timestamp, actual.Timestamp) } diff --git a/tools/querytee/response_comparator_test.go b/tools/querytee/response_comparator_test.go index 4603c087a3b..a9ec0a1565d 100644 --- a/tools/querytee/response_comparator_test.go +++ b/tools/querytee/response_comparator_test.go @@ -1139,6 +1139,7 @@ func TestCompareSamplesResponse(t *testing.T) { err error useRelativeError bool skipRecentSamples time.Duration + skipSamplesBefore int64 // In unix milliseconds }{ { name: "difference in response status", @@ -2166,12 +2167,259 @@ func TestCompareSamplesResponse(t *testing.T) { }`), err: errors.New(`expected info annotations ["\"info\" #1"] but got ["\"info\" #2"]`), }, + { + name: "should not fail when we skip samples from the beginning of a matrix for expected and actual - float", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"matrix","result":[{"metric":{"foo":"bar"},"values":[[90,"9"], [100,"10"]]}, {"metric":{"foo":"bar2"},"values":[[100,"10"]]}]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"matrix","result":[{"metric":{"foo":"bar"},"values":[[100,"10"]]}, {"metric":{"foo":"bar2"},"values":[[80,"9"], [100,"10"]]}]} + }`), + skipSamplesBefore: 95 * 1000, + }, + { + name: "should not fail when we skip all samples starting from the beginning - float", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"matrix","result":[{"metric":{"foo":"bar"},"values":[[90,"9"], [100,"10"]]}]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"matrix","result":[{"metric":{"foo":"bar"},"values":[[100,"10"]]}]} + }`), + skipSamplesBefore: 105 * 1000, + }, + { + name: "should fail when we skip partial samples in the beginning but compare some other - float", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"matrix","result":[{"metric":{"foo":"bar"},"values":[[90,"9"], [97,"7"], [100,"10"]]}, {"metric":{"foo":"bar2"},"values":[[100,"10"]]}]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"matrix","result":[{"metric":{"foo":"bar"},"values":[[100,"10"]]}, {"metric":{"foo":"bar2"},"values":[[90,"10"], [100,"10"]]}]} + }`), + skipSamplesBefore: 95 * 1000, + // 9 @[90] is not compared. foo=bar2 does not fail. + err: errors.New(`float sample pair does not match for metric {foo="bar"}: expected timestamp 97 but got 100 +Expected result for series: +{foo="bar"} => +7 @[97] +10 @[100] + +Actual result for series: +{foo="bar"} => +10 @[100]`), + }, + { + name: "should not fail when we skip samples from the beginning of a matrix for expected and actual - histogram", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"matrix","result":[ + {"metric":{"foo":"bar"},"histograms":[[90,{"count": "2","sum": "4","buckets": [[1,"0","2","2"]]}], [100,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]]}, + {"metric":{"foo":"bar2"},"histograms":[[100,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]]}]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"matrix","result":[ + {"metric":{"foo":"bar"},"histograms":[[100,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]]}, + {"metric":{"foo":"bar2"},"histograms":[[80,{"count": "2","sum": "4","buckets": [[1,"0","2","2"]]}], [100,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]]}]} + }`), + skipSamplesBefore: 95 * 1000, + }, + { + name: "should not fail when we skip all samples starting from the beginning - histogram", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"matrix","result":[{"metric":{"foo":"bar"},"histograms":[[90,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}], [100,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]]}]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"matrix","result":[{"metric":{"foo":"bar"},"histograms":[[100,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]]}]} + }`), + skipSamplesBefore: 105 * 1000, + }, + { + name: "should fail when we skip partial samples in the beginning but compare some other - histogram", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"matrix","result":[ + {"metric":{"foo":"bar"},"histograms":[[90,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}], [97,{"count": "2","sum": "33","buckets": [[1,"0","2","2"]]}], [100,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]]}, + {"metric":{"foo":"bar2"},"histograms":[[100,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]]}]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"matrix","result":[ + {"metric":{"foo":"bar"},"histograms":[[100,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]]}, + {"metric":{"foo":"bar2"},"histograms":[[90,{"count": "2","sum": "44","buckets": [[1,"0","2","2"]]}], [100,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]]}]} + }`), + skipSamplesBefore: 95 * 1000, + // @[90] is not compared. foo=bar2 does not fail. + err: errors.New(`histogram sample pair does not match for metric {foo="bar"}: expected timestamp 97 but got 100 +Expected result for series: +{foo="bar"} => +Count: 2.000000, Sum: 33.000000, Buckets: [[0,2):2] @[97] +Count: 2.000000, Sum: 3.000000, Buckets: [[0,2):2] @[100] + +Actual result for series: +{foo="bar"} => +Count: 2.000000, Sum: 3.000000, Buckets: [[0,2):2] @[100]`), + }, + { + name: "should not fail when skipped samples properly for vectors", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[90,"1"]}]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[95,"1"]}]} + }`), + skipSamplesBefore: 100 * 1000, + }, + { + name: "should not fail when skipped samples properly for vectors - histogram", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"histogram":[90,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]}]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"histogram":[95,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]}]} + }`), + skipSamplesBefore: 100 * 1000, + }, + { + name: "should fail when skipped samples only for expected vector", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[90,"1"]}]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[105,"1"]}]} + }`), + skipSamplesBefore: 100 * 1000, + err: errors.New(`expected 0 metrics but got 1 (also, some samples were filtered out from the expected response due to the 'skip samples before'; if all samples have been filtered out, this could cause the check on the expected number of metrics to fail)`), + }, + { + name: "should fail when skipped samples only for actual vector", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[105,"1"]}]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[95,"1"]}]} + }`), + skipSamplesBefore: 100 * 1000, + err: errors.New(`expected 1 metrics but got 0 (also, some samples were filtered out from the actual response due to the 'skip samples before'; if all samples have been filtered out, this could cause the check on the expected number of metrics to fail)`), + }, + { + name: "should skip properly when there are multiple series in a vector", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[90,"1"]}, {"metric":{"foo":"bar2"},"value":[105,"1"]}]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[95,"1"]}, {"metric":{"foo":"bar2"},"value":[105,"1"]}]} + }`), + skipSamplesBefore: 100 * 1000, + }, + { + name: "should skip properly when there are multiple series in a vector - histogram", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"vector","result":[ + {"metric":{"foo":"bar"},"histogram":[90,{"count":"2","sum":"333","buckets":[[1,"0","2","2"]]}]}, + {"metric":{"foo":"bar2"},"histogram":[105,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]}]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"vector","result":[ + {"metric":{"foo":"bar"},"histogram":[95,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]}, + {"metric":{"foo":"bar2"},"histogram":[105,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]}]} + }`), + skipSamplesBefore: 100 * 1000, + }, + { + name: "different series skipped in expected and actual, causing an error", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[105,"1"]}, {"metric":{"foo":"bar2"},"value":[90,"1"]}]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[95,"1"]}, {"metric":{"foo":"bar2"},"value":[105,"1"]}]} + }`), + skipSamplesBefore: 100 * 1000, + err: errors.New(`expected metric {foo="bar"} missing from actual response (also, some samples were filtered out from the expected and actual response due to the 'skip samples before'; if all samples have been filtered out, this could cause the check on the expected number of metrics to fail)`), + }, + { + name: "different series skipped in expected and actual, causing an error - histogram", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"vector","result":[ + {"metric":{"foo":"bar"},"histogram":[105,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]}, + {"metric":{"foo":"bar2"},"histogram":[90,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]}]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"vector","result":[ + {"metric":{"foo":"bar"},"histogram":[95,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]}, + {"metric":{"foo":"bar2"},"histogram":[105,{"count":"2","sum":"3","buckets":[[1,"0","2","2"]]}]}]} + }`), + skipSamplesBefore: 100 * 1000, + err: errors.New(`expected metric {foo="bar"} missing from actual response (also, some samples were filtered out from the expected and actual response due to the 'skip samples before'; if all samples have been filtered out, this could cause the check on the expected number of metrics to fail)`), + }, + { + name: "expected is skippable but not the actual, causing an error", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"scalar","result":[90,"1"]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"scalar","result":[100,"1"]} + }`), + skipSamplesBefore: 95 * 1000, + err: errors.New(`expected timestamp 90 but got 100`), + }, + { + name: "actual is skippable but not the expected, causing an error", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"scalar","result":[100,"1"]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"scalar","result":[90,"1"]} + }`), + skipSamplesBefore: 95 * 1000, + err: errors.New(`expected timestamp 100 but got 90`), + }, + { + name: "both expected and actual are skippable", + expected: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"scalar","result":[95,"1"]} + }`), + actual: json.RawMessage(`{ + "status": "success", + "data": {"resultType":"scalar","result":[90,"1"]} + }`), + skipSamplesBefore: 100 * 1000, + }, } { t.Run(tc.name, func(t *testing.T) { samplesComparator := NewSamplesComparator(SampleComparisonOptions{ Tolerance: tc.tolerance, UseRelativeError: tc.useRelativeError, SkipRecentSamples: tc.skipRecentSamples, + SkipSamplesBefore: model.Time(tc.skipSamplesBefore), }) result, err := samplesComparator.Compare(tc.expected, tc.actual, nowT.Time()) if tc.err == nil {