From 4305ce3061d3235302a904fd679f0b31ac4b363a Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Thu, 19 Dec 2024 15:37:33 +0000 Subject: [PATCH 1/5] Mark most client.geo.* as dynamic in TestRUMXForwardedFor approvals (#14997) Mark client.geo.{city_name,location,region_iso_code,region_name} as dynamic in TestRUMXforwardedFor. This will make it consistent with other systemtest approvals. --- .../approvals/TestRUMXForwardedFor.approved.json | 12 ++++++------ systemtest/rum_test.go | 3 +++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/systemtest/approvals/TestRUMXForwardedFor.approved.json b/systemtest/approvals/TestRUMXForwardedFor.approved.json index 89d28519fa6..f1b96a8c854 100644 --- a/systemtest/approvals/TestRUMXForwardedFor.approved.json +++ b/systemtest/approvals/TestRUMXForwardedFor.approved.json @@ -13,7 +13,7 @@ "5.5.0" ], "client.geo.city_name": [ - "Perth" + "dynamic" ], "client.geo.continent_name": [ "Oceania" @@ -28,10 +28,10 @@ "dynamic" ], "client.geo.region_iso_code": [ - "AU-WA" + "dynamic" ], "client.geo.region_name": [ - "Western Australia" + "dynamic" ], "client.ip": [ "220.244.41.16" @@ -132,7 +132,7 @@ "5.5.0" ], "client.geo.city_name": [ - "Perth" + "dynamic" ], "client.geo.continent_name": [ "Oceania" @@ -147,10 +147,10 @@ "dynamic" ], "client.geo.region_iso_code": [ - "AU-WA" + "dynamic" ], "client.geo.region_name": [ - "Western Australia" + "dynamic" ], "client.ip": [ "220.244.41.16" diff --git a/systemtest/rum_test.go b/systemtest/rum_test.go index d48659dc205..db5af921e46 100644 --- a/systemtest/rum_test.go +++ b/systemtest/rum_test.go @@ -78,7 +78,10 @@ func TestRUMXForwardedFor(t *testing.T) { "source.port", // Do not assert the exact contents of the location field since they may change // slightly depending on the IP lookup. + "client.geo.city_name", "client.geo.location", + "client.geo.region_iso_code", + "client.geo.region_name", ) } From 1ea743952f24a239207b5d3deb2653b53f958b87 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Thu, 19 Dec 2024 16:27:17 +0000 Subject: [PATCH 2/5] Record tbs disk usage stats in benchtest (#14995) Record TBS lsm size and vlog size in benchtest to facilitate TBS improvements when running benchmarks using gh actions following #14985 --- systemtest/benchtest/expvar/expvar.go | 12 ++++++++++++ systemtest/benchtest/expvar/metrics.go | 4 ++++ systemtest/benchtest/main.go | 2 ++ systemtest/benchtest/main_test.go | 4 ++++ 4 files changed, 22 insertions(+) diff --git a/systemtest/benchtest/expvar/expvar.go b/systemtest/benchtest/expvar/expvar.go index 452e3ddf0bf..be407e1f98a 100644 --- a/systemtest/benchtest/expvar/expvar.go +++ b/systemtest/benchtest/expvar/expvar.go @@ -36,6 +36,7 @@ type expvar struct { LibbeatStats ElasticResponseStats OTLPResponseStats + TailSamplingStats // UncompressedBytes holds the number of bytes of uncompressed // data that the server has read from the Elastic APM events @@ -72,6 +73,11 @@ type LibbeatStats struct { RSSMemoryBytes int64 `json:"beat.memstats.rss"` } +type TailSamplingStats struct { + TBSLsmSize int64 `json:"apm-server.sampling.tail.storage.lsm_size"` + TBSVlogSize int64 `json:"apm-server.sampling.tail.storage.value_log_size"` +} + func queryExpvar(ctx context.Context, out *expvar, srv string) error { req, err := http.NewRequest("GET", srv+"/debug/vars", nil) if err != nil { @@ -113,6 +119,7 @@ func queryExpvar(ctx context.Context, out *expvar, srv string) error { aggregateResponseStats(s.ElasticResponseStats, &result.ElasticResponseStats) aggregateOTLPResponseStats(s.OTLPResponseStats, &result.OTLPResponseStats) aggregateLibbeatStats(s.LibbeatStats, &result.LibbeatStats) + aggregateTailSamplingStats(s.TailSamplingStats, &result.TailSamplingStats) result.UncompressedBytes += s.UncompressedBytes result.AvailableBulkRequests += s.AvailableBulkRequests } @@ -205,3 +212,8 @@ func aggregateOTLPResponseStats(from OTLPResponseStats, to *OTLPResponseStats) { to.ErrorOTLPTracesResponses += from.ErrorOTLPTracesResponses to.ErrorOTLPMetricsResponses += from.ErrorOTLPMetricsResponses } + +func aggregateTailSamplingStats(from TailSamplingStats, to *TailSamplingStats) { + to.TBSLsmSize += from.TBSLsmSize + to.TBSVlogSize += from.TBSVlogSize +} diff --git a/systemtest/benchtest/expvar/metrics.go b/systemtest/benchtest/expvar/metrics.go index 328c78bd9a5..c21bd0db051 100644 --- a/systemtest/benchtest/expvar/metrics.go +++ b/systemtest/benchtest/expvar/metrics.go @@ -47,6 +47,8 @@ const ( ErrorElasticResponses ErrorOTLPTracesResponses ErrorOTLPMetricsResponses + TBSLsmSize + TBSVlogSize ) type AggregateStats struct { @@ -164,6 +166,8 @@ func (c *Collector) accumulate(e expvar) { c.processMetric(MemBytes, int64(e.TotalAlloc)) c.processMetric(HeapAlloc, int64(e.HeapAlloc)) c.processMetric(HeapObjects, int64(e.HeapObjects)) + c.processMetric(TBSLsmSize, e.TBSLsmSize) + c.processMetric(TBSVlogSize, e.TBSVlogSize) } func (c *Collector) processMetric(m Metric, val int64) { diff --git a/systemtest/benchtest/main.go b/systemtest/benchtest/main.go index 5422915b491..abab1cceb2f 100644 --- a/systemtest/benchtest/main.go +++ b/systemtest/benchtest/main.go @@ -122,6 +122,8 @@ func addExpvarMetrics(result *testing.BenchmarkResult, collector *expvar.Collect result.Extra["max_heap_alloc"] = float64(collector.Get(expvar.HeapAlloc).Max) result.Extra["max_heap_objects"] = float64(collector.Get(expvar.HeapObjects).Max) result.Extra["mean_available_indexers"] = float64(collector.Get(expvar.AvailableBulkRequests).Mean) + result.Extra["tbs_lsm_size"] = float64(collector.Get(expvar.TBSLsmSize).Max) + result.Extra["tbs_vlog_size"] = float64(collector.Get(expvar.TBSVlogSize).Max) } // Record the number of error responses returned by the server: lower is better. diff --git a/systemtest/benchtest/main_test.go b/systemtest/benchtest/main_test.go index 14a7bbd64b9..a94c6f62e8f 100644 --- a/systemtest/benchtest/main_test.go +++ b/systemtest/benchtest/main_test.go @@ -141,6 +141,8 @@ func TestAddExpvarMetrics(t *testing.T) { `"apm-server.processor.span.transformations": 5`, `"apm-server.processor.metric.transformations": 9`, `"apm-server.processor.error.transformations": 3`, + `"apm-server.sampling.tail.storage.lsm_size": 10`, + `"apm-server.sampling.tail.storage.value_log_size": 11`, `"beat.runtime.goroutines": 4`, `"beat.memstats.rss": 1048576`, `"output.elasticsearch.bulk_requests.available": 0`, @@ -165,6 +167,8 @@ func TestAddExpvarMetrics(t *testing.T) { "max_heap_objects": 102, "mean_available_indexers": 0, "error_responses/sec": 1, + "tbs_lsm_size": 10, + "tbs_vlog_size": 11, }, }, } From 7c1b7060db534812946461d2e8bf2b9f8778237b Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Thu, 19 Dec 2024 17:46:59 +0000 Subject: [PATCH 3/5] Fix TF_VAR_apm_server_tail_sampling for scheduled gh actions (#15003) A regression from #14985 causing nightly benchmark to fail. --- .github/workflows/benchmarks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 6f7fadfee91..ed251f5a768 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -62,8 +62,8 @@ jobs: TF_VAR_private_key: ./id_rsa_terraform TF_VAR_public_key: ./id_rsa_terraform.pub TF_VAR_run_standalone: ${{ inputs.runStandalone || github.event.schedule=='0 5 */5 * *' }} - TF_VAR_apm_server_tail_sampling: ${{ inputs.enableTailSampling }} - TF_VAR_apm_server_tail_sampling_storage_limit: ${{ inputs.tailSamplingStorageLimit }} + TF_VAR_apm_server_tail_sampling: ${{ inputs.enableTailSampling || 'false' }} # set the default again otherwise schedules won't work + TF_VAR_apm_server_tail_sampling_storage_limit: ${{ inputs.tailSamplingStorageLimit || '10GB' }} # set the default again otherwise schedules won't work RUN_STANDALONE: ${{ inputs.runStandalone || github.event.schedule=='0 5 */5 * *' }} TFVARS_SOURCE: ${{ inputs.profile || 'system-profiles/8GBx1zone.tfvars' }} # // Default to use an 8gb profile TF_VAR_BUILD_ID: ${{ github.run_id }} From a261e20958296770c0cf7095926201b0133001fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Dec 2024 15:37:04 +0000 Subject: [PATCH 4/5] build(deps): bump ubi9/ubi-micro in /packaging/docker (#14966) Bumps ubi9/ubi-micro from `a410623` to `a22fffe`. --- updated-dependencies: - dependency-name: ubi9/ubi-micro dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packaging/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index c9a4b0851ed..c0d10876a50 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -30,7 +30,7 @@ RUN sed -i 's/localhost:9200/elasticsearch:9200/' apm-server.yml # Build stage 1 # Copy prepared files from the previous stage and complete the image. ################################################################################ -FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:a410623c2b8e9429f9606af821be0231fef2372bd0f5f853fbe9743a0ddf7b34 +FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:a22fffe0256af00176c8b4f22eec5d8ecb1cb1684d811c33b1f2832fd573260f ARG TARGETARCH ARG BUILD_DATE ARG VERSION From 8d40b5e95851c9b30a0673548946ab23785fc761 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Dec 2024 15:48:55 +0000 Subject: [PATCH 5/5] build(deps): bump golang.org/x/net from 0.32.0 to 0.33.0 in the golang-org-x group (#15005) * build(deps): bump golang.org/x/net in the golang-org-x group Bumps the golang-org-x group with 1 update: [golang.org/x/net](https://github.com/golang/net). Updates `golang.org/x/net` from 0.32.0 to 0.33.0 - [Commits](https://github.com/golang/net/compare/v0.32.0...v0.33.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor dependency-group: golang-org-x ... Signed-off-by: dependabot[bot] * Update NOTICE.txt --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: kruskall <99559985+kruskall@users.noreply.github.com> --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index ca7a1fe64aa..d04e38ed613 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -5706,11 +5706,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.32.0 +Version: v0.33.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.32.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.33.0/LICENSE: Copyright 2009 The Go Authors. diff --git a/go.mod b/go.mod index a8e988f58fa..6df841ba890 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( go.uber.org/automaxprocs v1.6.0 go.uber.org/zap v1.27.0 go.uber.org/zap/exp v0.3.0 - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 golang.org/x/sync v0.10.0 golang.org/x/term v0.27.0 golang.org/x/time v0.8.0 diff --git a/go.sum b/go.sum index b8f87b4cc3b..6cb40beb91e 100644 --- a/go.sum +++ b/go.sum @@ -522,8 +522,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=