diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile deleted file mode 100644 index 7a79def7..00000000 --- a/.ci/Jenkinsfile +++ /dev/null @@ -1,302 +0,0 @@ -#!/usr/bin/env groovy - -@Library('apm@current') _ - -pipeline { - agent { label 'ubuntu-20.04 && immutable' } - environment { - REPO = "synthetics-recorder" - BASE_DIR = "src/github.com/elastic/${env.REPO}" - DOCKER_REGISTRY = 'docker.elastic.co' - DOCKER_ELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' - JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba" - PIPELINE_LOG_LEVEL = "INFO" - NOTIFY_TO = 'synthrum+synthetics-recorder@elastic.co' - SLACK_CHANNEL = '#observablt-bots' - } - options { - timeout(time: 2, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) - timestamps() - ansiColor('xterm') - disableResume() - durabilityHint('PERFORMANCE_OPTIMIZED') - rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) - quietPeriod(10) - } - stages { - stage('Checkout') { - options { skipDefaultCheckout() } - steps { - whenTrue(isInternalCI() && isTag()) { - notifyStatus(slackStatus: 'good', subject: "[${env.REPO}] Build for the release tag *${env.TAG_NAME}* has been triggered", body: "Build: (<${env.RUN_DISPLAY_URL}|here>) for further details.") - } - pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ]) - deleteDir() - gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true) - stash(allowEmpty: true, name: 'source', useDefaultExcludes: false) - // Fail fast if no access to docker login - retryWithSleep(retries: 3, seconds: 10, backoff: true) { - dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") - } - // Fail fast if no access to AWS - dir("${BASE_DIR}") { - withS3Creds { - sh(label: 'list-s3-artifacts.sh', script: '.ci/scripts/list-s3-artifacts.sh') - } - } - } - } - stage('Release') { - options { skipDefaultCheckout() } - when { - beforeAgent true - allOf { - tag pattern: 'v\\d+\\.\\d+.*', comparator: 'REGEXP' - expression { isInternalCI() } - } - } - environment { - BUCKET_NAME = 'internal-ci-artifacts' - BUCKET_SUBFOLDER = "${env.REPO}/${env.TAG_NAME}/${env.BUILD_ID}" - BUCKET_PATH = "gs://${env.BUCKET_NAME}/${env.BUCKET_SUBFOLDER}" - BUCKET_CREDENTIALS = 'internal-ci-gcs-plugin' - SIGNED_ARTIFACTS = 'signed-artifacts' - BUCKET_SUBFOLDER_SIGNED_ARTIFACTS = "${env.BUCKET_SUBFOLDER}/${env.SIGNED_ARTIFACTS}" - BUCKET_SIGNED_ARTIFACTS_PATH = "gs://${env.BUCKET_NAME}/${env.BUCKET_SUBFOLDER_SIGNED_ARTIFACTS}" - DIST_FOLDER = 'dist' - RELEASE_URL_MESSAGE = "()" - } - // There are some environmental issues with some of the CI workers - // let's use the one we know it works. - // agent { label 'macos-code-signer' } - agent { label 'worker-c07yc0cejyvy' } - stages { - stage('Dist') { - options { skipDefaultCheckout() } - steps { - deleteDir() - unstash 'source' - withNodeJSEnv() { - dir("${BASE_DIR}"){ - runReleaseWithKeychain() - } - } - } - post { - success { - dir("${BASE_DIR}"){ - stash(allowEmpty: false, name: 'exe', includes: "${env.DIST_FOLDER}/*.exe") - } - } - } - } - stage('Signing Windows') { - agent { label 'cloud-hsm' } - options { skipDefaultCheckout() } - environment { - BASE_DIR_WIN = "${BASE_DIR.replaceAll('\\/', '\\\\')}" - } - steps { - deleteDir() - unstash 'source' - unstash('exe') - withAzureSigningCreds { - dir("${env.DIST_FOLDER}") { - getFirstExeFileName() - bat(label: 'ACS signing', script: "${env.WORKSPACE}\\${BASE_DIR_WIN}\\.ci\\scripts\\acs-signing.bat ${env.WORKSPACE}\\${env.DIST_FOLDER}\\${env.EXE_FILE_NAME}") - stash(includes: "${env.EXE_FILE_NAME}", name: 'signed_exe', allowEmpty: false) - } - } - } - post { - cleanup { - deleteDir() - } - } - } - stage('Signing *Nix') { - options { skipDefaultCheckout() } - steps { - dir("${BASE_DIR}") { - uploadBinariesToBeSigned(['deb', 'dmg', 'zip']) - build(job: 'elastic+unified-release+master+sign-artifacts-with-gpg', - parameters: [string(name: 'gcs_input_path', value: "${env.BUCKET_PATH}")], - wait: true, - propagate: true) - } - } - } - stage('Download signed artifacts') { - options { skipDefaultCheckout() } - steps { - dir("${BASE_DIR}/${SIGNED_ARTIFACTS}") { - downloadSignedBinaries() - } - } - } - stage('Publish S3 Artifacts') { - options { skipDefaultCheckout() } - agent { label 'ubuntu-20.04 && immutable' } // Required to run within the context of an ephemeral worker so docker is available - steps { - deleteDir() - retryWithSleep(retries: 3, seconds: 10, backoff: true) { - dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") - } - unstash 'source' - dir("${BASE_DIR}/${SIGNED_ARTIFACTS}") { - downloadSignedBinaries() - } - dir("${BASE_DIR}") { - withS3Creds { - sh(label: 'publish-s3-artifacts.sh', script: '.ci/scripts/publish-s3-artifacts.sh "${SIGNED_ARTIFACTS}/${DIST_FOLDER}"') - } - } - } - } - stage('Publish GitHub Release') { - options { skipDefaultCheckout() } - steps { - dir("${BASE_DIR}") { - withCredentials([string(credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7', variable: 'GITHUB_TOKEN')]) { - sh(label: 'create-github-release.sh', script: '.ci/scripts/create-github-release.sh "${SIGNED_ARTIFACTS}/${DIST_FOLDER}"') - } - } - } - } - } - post { - success { - notifyStatus(slackStatus: 'good', subject: "[${env.REPO}] Release *${env.TAG_NAME}* published", body: "Build: (<${env.RUN_DISPLAY_URL}|here>), Github Release: ${RELEASE_URL_MESSAGE}") - } - failure { - notifyStatus(slackStatus: 'warning', subject: "[${env.REPO}] Release *${env.TAG_NAME}* could not be published", body: "Build: (<${env.RUN_DISPLAY_URL}|here>)") - } - cleanup { - deleteDir() - } - } - } - } -} - -def notifyStatus(def args = [:]) { - releaseNotification(slackChannel: "${env.SLACK_CHANNEL}", - slackColor: args.slackStatus, - slackCredentialsId: 'jenkins-slack-integration-token', - to: "${env.NOTIFY_TO}", - subject: args.subject, - body: args.body) -} - -def downloadSignedBinaries() { - googleStorageDownload(bucketUri: "${env.BUCKET_SIGNED_ARTIFACTS_PATH}/*", - credentialsId: env.BUCKET_CREDENTIALS, - localDirectory: "${DIST_FOLDER}/", - pathPrefix: "${env.BUCKET_SUBFOLDER_SIGNED_ARTIFACTS}") - dir("${env.DIST_FOLDER}") { - unstash 'signed_exe' - } - archiveArtifacts(allowEmptyArchive: false, artifacts: "${DIST_FOLDER}/*.*") -} - -def uploadBinariesToBeSigned(extensions = [:]) { - extensions?.each { extension -> - googleStorageUpload(bucket: env.BUCKET_PATH, - credentialsId: env.BUCKET_CREDENTIALS, - pathPrefix: "${env.DIST_FOLDER}/", - pattern: "${env.DIST_FOLDER}/*.${extension}", - sharedPublicly: false, - showInline: true) - } -} - -def runReleaseWithKeychain() { - def vault = getVaultSecret(secret: 'secret/observability-team/ci/synthetics-recorder-macos-vault').data - if (!vault.containsKey('vault_secret_id') || !vault.containsKey('vault_role_id') || !vault.containsKey('vault_addr')) { - error("runReleaseWithKeychain: macos secret could not be accessed correctly") - } - def token = getVaultSecret.getVaultToken(vault.vault_addr, vault.vault_role_id, vault.vault_secret_id) - def apple = getVaultSecret.getVaultSecretObject(vault.vault_addr, 'secret/release/apple-2023', token).data - if (!apple.containsKey('keychain-password') || !apple.containsKey('username') || !apple.containsKey('app-specific-password')) { - error("runReleaseWithKeychain: apple secret must contain username, app-specific-password") - } - withEnvMask(vars: [ - [var: 'KEYCHAIN_PASSWORD', password: apple.get('keychain-password')], - [var: 'APPLE_USERNAME', password: apple.get('username')], - [var: 'APPLE_APP_SPECIFIC_PASSWORD', password: apple.get('app-specific-password')], - ]){ - sh(label: 'release-ci.sh', script: '.ci/scripts/release-ci.sh') - } -} - -def withAzureSigningCreds(Closure body) { - def syntheticsRecorderWindowsVault = 'secret/observability-team/ci/synthetics-recorder-windows-vault' - def vault = getVaultSecret(secret: syntheticsRecorderWindowsVault).data - if (!vault.containsKey('vault_secret_id') || !vault.containsKey('vault_role_id') || !vault.containsKey('vault_addr')) { - error("withAzureSigningCreds: ${syntheticsRecorderWindowsVault} secert must contain vault_secret_id, vault_role_id and vault_addr entries") - } - def token = getVaultSecret.getVaultToken(vault.vault_addr, vault.vault_role_id, vault.vault_secret_id) - def cloudHsmPath = 'secret/release/cloud-hsm' - def azureSecrets = getVaultSecret.getVaultSecretObject(vault.vault_addr, cloudHsmPath, token).data - if (!azureSecrets.containsKey('azure-tenant-id') || !azureSecrets.containsKey('azure-client-id') || !azureSecrets.containsKey('azure-client-secret')) { - error("withAzureSigningCreds: ${cloudHsmPath} must contain azure-tenant-id, azure-client-id and azure-client-secret entries") - } - - withEnvMask(vars: [ - [var: "AZURE_TENANT_ID", password: azureSecrets['azure-tenant-id']], - [var: "AZURE_CLIENT_ID", password: azureSecrets['azure-client-id']], - [var: "AZURE_CLIENT_SECRET", password: azureSecrets['azure-client-secret']] - ]) { - body() - } -} - -/* -Sets EXE_FILE_NAME env var as first found exe file in current dir -*/ -def getFirstExeFileName() { - def files = findFiles(glob: '*.exe') - env.EXE_FILE_NAME = "${files[0].name}" -} - -def withS3Creds(Closure body) { - def vault = getVaultSecret(secret: 'secret/observability-team/ci/synthetics-recorder-macos-vault').data - if (!vault.containsKey('vault_secret_id') || !vault.containsKey('vault_role_id') || !vault.containsKey('vault_addr')) { - error("withS3Creds: s3 secret could not be accessed correctly") - } - def token = getVaultSecret.getVaultToken(vault.vault_addr, vault.vault_role_id, vault.vault_secret_id) - def s3 = readAWS(vault.vault_addr, 'aws-elastic/sts/synthetics-recorder', token).data - if (!s3.containsKey('access_key') || !s3.containsKey('secret_key')) { - error("withS3Creds: s3 secret must contain access_key and secret_key") - } - - withEnvMask(vars: [ - [var: 'AWS_ACCESS_KEY_ID', password: s3.get('access_key')], - [var: 'AWS_SECRET_ACCESS_KEY', password: s3.get('secret_key')], - ]){ - // Fixes An error occurred (InvalidAccessKeyId) when calling the ListObjectsV2 operation: The AWS Access Key Id you provided does not exist in our records. - log(level: 'INFO', text: "As long as vault secrets are not synced let's wait for a few seconds until they are available") - sleep 15 - body() - } -} - -def readAWS(addr, secret, token) { - wrap([$class: 'MaskPasswordsBuildWrapper', varPasswordPairs: [ - [var: 'VAULT_SECRET', password: secret], - [var: 'VAULT_TOKEN', password: token], - [var: 'VAULT_ADDR', password: addr], - ]]) { - // sts secrets use write rather than read - // vault write -f - def retJson = httpRequest(url: "${addr}/v1/${secret}", - method: 'POST', - headers: ['X-Vault-Token': "${token}", 'Content-Type': 'application/json']) - def obj = toJSON(retJson) - if(!(obj instanceof net.sf.json.JSONObject)){ - error("readAWS: Unable to get the secret.") - } - return obj - } -} diff --git a/.ci/scripts/acs-signing.bat b/.ci/scripts/acs-signing.bat deleted file mode 100644 index 7b770301..00000000 --- a/.ci/scripts/acs-signing.bat +++ /dev/null @@ -1 +0,0 @@ -c:\tools\Microsoft.Windows.SDK.BuildTools.10.0.20348.19\bin\10.0.20348.0\x64\signtool.exe sign /v /debug /fd sha256 /tr "http://timestamp.acs.microsoft.com" /td sha256 /as /ph /dmdf "C:\tools\Azure.CodeSigning.Dlib.1.0.19\metadata.json" /dlib "C:\tools\Azure.CodeSigning.Dlib.1.0.19\bin\x64\Azure.CodeSigning.Dlib.dll" %* \ No newline at end of file diff --git a/.ci/scripts/create-github-release.sh b/.ci/scripts/create-github-release.sh deleted file mode 100755 index 75fb348b..00000000 --- a/.ci/scripts/create-github-release.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -set -eox pipefail - -DIST_LOCATION=$1 - -ls -l "${DIST_LOCATION}"/*.* || true - -if ! gh --version &>/dev/null ; then - ## install gh - wget -q https://github.com/cli/cli/releases/download/v2.4.0/gh_2.4.0_macOS_amd64.tar.gz -O gh.tar.gz - tar -xpf gh.tar.gz --strip-components=2 - PATH="$(pwd):${PATH}" - export PATH - gh --version -fi - -gh release \ - create \ - "${TAG_NAME}" \ - --draft \ - --title "${TAG_NAME}" \ - --repo "elastic/${REPO}" \ - "${DIST_LOCATION}"/*.* diff --git a/.ci/scripts/docker-build.sh b/.ci/scripts/docker-build.sh deleted file mode 100755 index 2eb7143d..00000000 --- a/.ci/scripts/docker-build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -set -eox pipefail - -TAG=${BRANCH_NAME:-'latest'} - -# Read node version from .nvmrc and remove v prefix. -NODE_VERSION=$(cat .nvmrc | sed "s#v##g") - -# Download the docker image if possible -docker pull docker.elastic.co/observability-ci/synthetics-recorder:$TAG || true - -docker \ - build \ - --file e2e/Dockerfile.jenkins \ - --build-arg DISPLAY=":99" \ - --build-arg NODE_VERSION="${NODE_VERSION}" \ - --build-arg UID="$UID" \ - --tag docker.elastic.co/observability-ci/synthetics-recorder:$TAG . diff --git a/.ci/scripts/docker-push.sh b/.ci/scripts/docker-push.sh deleted file mode 100755 index e6bae63b..00000000 --- a/.ci/scripts/docker-push.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -eox pipefail - -TAG=${BRANCH_NAME:-'latest'} - -docker \ - push \ - docker.elastic.co/observability-ci/synthetics-recorder:$TAG diff --git a/.ci/scripts/list-s3-artifacts.sh b/.ci/scripts/list-s3-artifacts.sh deleted file mode 100755 index 48c21055..00000000 --- a/.ci/scripts/list-s3-artifacts.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -echo "> List artifacts from S3" -docker run \ - -e AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY \ - docker.elastic.co/infra/aws-cli:latest \ - aws s3 ls s3://download.elasticsearch.org/synthetics-recorder/ diff --git a/.ci/scripts/publish-s3-artifacts.sh b/.ci/scripts/publish-s3-artifacts.sh deleted file mode 100755 index 1ad038cf..00000000 --- a/.ci/scripts/publish-s3-artifacts.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -DIST_LOCATION=$1 -DOCKER_IMAGE=docker.elastic.co/infra/aws-cli:latest -S3_PATH=s3://download.elasticsearch.org/synthetics-recorder/ - -echo "> List artifacts in the dist folder ($DIST_LOCATION)" -ls -l "${DIST_LOCATION}"/*.* || true - -echo "Fetch docker image" -docker pull $DOCKER_IMAGE --quiet - -echo "> Publish artifacts to S3" -cd "${DIST_LOCATION}" -for artifact in ./*; do - echo ">> publishing $artifact ..." - docker run \ - -e AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY \ - -v "$(pwd)":/root \ - $DOCKER_IMAGE \ - aws s3 cp /root/"$artifact" $S3_PATH -done - -echo "> List artifacts from S3" -docker run \ - -e AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY \ - $DOCKER_IMAGE \ - aws s3 ls $S3_PATH diff --git a/.ci/scripts/release-ci.sh b/.ci/scripts/release-ci.sh deleted file mode 100755 index 5eb1f915..00000000 --- a/.ci/scripts/release-ci.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash --login - -set -eo pipefail - -echo 'set env context' -export HOME='/var/lib/jenkins' -export CSC_KEYCHAIN="/var/lib/jenkins/Library/Keychains/Elastic.keychain-db" - -echo 'set keychain' -set -x -security list-keychains -s "$CSC_KEYCHAIN" -security list-keychains - -set +x -echo 'unlock keychain' -security unlock-keychain -p "$KEYCHAIN_PASSWORD" "$CSC_KEYCHAIN" - -echo 'run release-ci' -set -x -npm ci - -# temp, TEAM_ID is public info that is available in codesign verification -export APPLE_TEAM_ID='2BT3HPN62Z' -export APPLE_USERNAME -export APPLE_APP_SPECIFIC_PASSWORD -npm run release-ci diff --git a/.ci/scripts/run-test-in-docker.sh b/.ci/scripts/run-test-in-docker.sh deleted file mode 100755 index e3ef670c..00000000 --- a/.ci/scripts/run-test-in-docker.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -set -eox pipefail - -TAG=${BRANCH_NAME:-'latest'} - -DOCKER_IMAGE=docker.elastic.co/observability-ci/synthetics-recorder:$TAG -if [[ "$(docker images -q $DOCKER_IMAGE 2> /dev/null)" == "" ]]; then - if ! docker pull $DOCKER_IMAGE ; then - # pull the docker image from the target branch - DOCKER_IMAGE=docker.elastic.co/observability-ci/synthetics-recorder:$CHANGE_TARGET - docker pull $DOCKER_IMAGE - fi -fi - -DOCKER_RUN_OPTIONS="-i --rm" - -# Only allocate tty if we detect one -if [ -t 0 ] && [ -t 1 ]; then - DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -t" -fi - -# shellcheck disable=SC2086 -docker run \ - $DOCKER_RUN_OPTIONS \ - -v "$(pwd):/synthetics-recorder" \ - -e NPM_COMMAND=${1:-''} \ - $DOCKER_IMAGE \ - .ci/scripts/run-test.sh diff --git a/.ci/scripts/run-test.sh b/.ci/scripts/run-test.sh deleted file mode 100755 index bd5936fd..00000000 --- a/.ci/scripts/run-test.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -Xvfb ${DISPLAY} -screen 0 1024x768x16 & - -set -x - -# If NPM_COMMAND then run it. -if [ -n "${NPM_COMMAND}" ] ; then - npm ${NPM_COMMAND} -fi -NPM_CONFIG_LOGLEVEL=verbose npm run test