[CI] Add weekly job to test packages with Ubuntu Elastic Agent docker…

… image (#10844) Added a new CI pipeline that runs once per week. Builds from this pipeline will run the system tests of the packages with the Elastic Agent docker image based on Ubuntu.
elastic · Aug 29, 2024 · 56a4c02 · 56a4c02
1 parent 91399a8
commit 56a4c02
Show file tree

Hide file tree

Showing 4 changed files with 111 additions and 5 deletions.
diff --git a/.buildkite/pipeline.schedule-weekly.yml b/.buildkite/pipeline.schedule-weekly.yml
@@ -0,0 +1,29 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+name: integrations-schedule-weekly
+
+env:
+  SETUP_GVM_VERSION: "v0.5.2"
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
+
+# The pipeline is triggered by the scheduler every week
+steps:
+  - label: ":white_check_mark: Check go sources"
+    key: "check"
+    command: ".buildkite/scripts/check_sources.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+
+  - label: "Check integrations local stacks and Elastic Agent Ubuntu docker - Stack Version v8.16"
+    trigger: "integrations"
+    build:
+      env:
+        SERVERLESS: "false"
+        FORCE_CHECK_ALL: "true"
+        STACK_VERSION: 8.16.0-SNAPSHOT
+        PUBLISH_COVERAGE_REPORTS: "false"
+        ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true"
+    depends_on:
+      - step: "check"
+        allow_failure: false
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -1,14 +1,16 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 env:
   SETUP_GVM_VERSION: "v0.5.2"
-  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
   DOCKER_COMPOSE_VERSION: "v2.24.1"
   DOCKER_VERSION: "26.1.2"
   KIND_VERSION: 'v0.20.0'
   K8S_VERSION: 'v1.31.0'
   YQ_VERSION: 'v4.35.2'
   JQ_VERSION: '1.7'
   GH_CLI_VERSION: "2.29.0"
+
+  # Agent images used in pipeline steps
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
 
   # Elastic package settings
@@ -24,6 +26,8 @@ env:
   ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true"
   # Set maximum number of parallel tests to run if package allows it
   ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5"
+  # Disable the usage of wolfi images for Elastic Agent
+  ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-true}"
 
 steps:
   - label: "Get reference from target branch"

diff --git a/.github/workflows/updatecli/updatecli.d/bump-latest-snapshot-version.yml b/.github/workflows/updatecli/updatecli.d/bump-latest-snapshot-version.yml
@@ -43,8 +43,8 @@ sources:
           captureindex: 1
 
 targets:
-  update-snapshot:
-    name: '[updatecli] Update latest snapshot to {{ source "latestSnapshot" }}'
+  update-snapshot-daily:
+    name: '[updatecli] [daily] Update latest snapshot to {{ source "latestSnapshot" }}'
     kind: file
     sourceid: latestSnapshot
     scmid: default
@@ -53,12 +53,34 @@ targets:
       matchpattern: '(STACK_VERSION:) 8\.[^\s]*\.[^\s]*'
       replacepattern: '$1 {{ source "latestSnapshot" }}'
 
-  update-snapshot-label:
-    name: '[updatecli] Update latest snapshot label step to {{ source "latestSnapshotMajorMinor" }}'
+  update-snapshot-label-daily:
+    name: '[updatecli] [daily] Update latest snapshot label step to {{ source "latestSnapshotMajorMinor" }}'
     kind: file
     sourceid: latestSnapshotMajorMinor
     scmid: default
     spec:
       file: '.buildkite/pipeline.schedule-daily.yml'
       matchpattern: '(Stack Version) v8\.\d+"'
       replacepattern: 'Stack Version v{{ source "latestSnapshotMajorMinor" }}"'
+
+  ## TODO: Looks like these targets are running with the contents from main branch
+  ## Uncomment this target once this code is merged
+  #update-snapshot-weekly:
+  #  name: '[updatecli] [weekly] Update latest snapshot to {{ source "latestSnapshot" }}'
+  #  kind: file
+  #  sourceid: latestSnapshot
+  #  scmid: default
+  #  spec:
+  #    file: '.buildkite/pipeline.schedule-weekly.yml'
+  #    matchpattern: '(STACK_VERSION:) 8\.[^\s]*\.[^\s]*'
+  #    replacepattern: '$1 {{ source "latestSnapshot" }}'
+
+  #update-snapshot-label-weekly:
+  #  name: '[updatecli] [weekly] Update latest snapshot label step to {{ source "latestSnapshotMajorMinor" }}'
+  #  kind: file
+  #  sourceid: latestSnapshotMajorMinor
+  #  scmid: default
+  #  spec:
+  #    file: '.buildkite/pipeline.schedule-weekly.yml'
+  #    matchpattern: '(Stack Version) v8\.\d+"'
+  #    replacepattern: 'Stack Version v{{ source "latestSnapshotMajorMinor" }}"'
diff --git a/catalog-info.yaml b/catalog-info.yaml
@@ -120,6 +120,57 @@ spec:
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1
 kind: Resource
+metadata:
+  name: pipeline-integrations-schedule-weekly
+  description: 'Weekly pipeline for the Integrations project'
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/integrations-schedule-weekly
+
+spec:
+  type: buildkite-pipeline
+  owner: group:ingest-fp
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: integrations-schedule-weekly
+      description: 'Weekly pipeline for the Integrations project'
+    spec:
+      pipeline_file: ".buildkite/pipeline.schedule-weekly.yml"
+      schedules:
+        main_weekly:
+          branch: "main"
+          cronline: "30 3 * * 1"
+          message: "Run the weekly jobs"
+      provider_settings:
+        trigger_mode: none # don't trigger jobs from github activity
+        build_pull_request_forks: false
+        build_pull_requests: true
+        build_tags: false
+        filter_enabled: true
+        filter_condition: >-
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null && build.source == 'api')
+      repository: elastic/integrations
+      cancel_intermediate_builds: true
+      cancel_intermediate_builds_branch_filter: '!main'
+      skip_intermediate_builds: true
+      skip_intermediate_builds_branch_filter: '!main'
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        SLACK_NOTIFICATIONS_CHANNEL: '#ingest-notifications'
+        SLACK_NOTIFICATIONS_ALL_BRANCHES: 'true'
+        SLACK_NOTIFICATIONS_ON_SUCCESS: 'true'
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
 metadata:
   name: pipeline-integrations-serverless
   description: 'Pipeline for the Integrations project in serverless'