From 4c77634d44803434d382ca223d8744de3400a934 Mon Sep 17 00:00:00 2001 From: Adrian Riobo Date: Mon, 21 Oct 2024 16:22:42 +0200 Subject: [PATCH 1/2] chore: migrate crc-builder-installer strategy to deliverest The current stratgy for crc-builder-installer had its own logic to run the builder scripts remotely. That logic was created before deliverest exists, as now we prefer to maintain it only from one project this commit remove legacy logic and move to use deliverest as base to run scripts remotely Signed-off-by: Adrian Riobo --- crc-builder/oci/Containerfile.non-linux | 23 +- crc-builder/oci/lib/common.sh | 83 ------- crc-builder/oci/lib/darwin/builder/build.sh | 109 --------- crc-builder/oci/lib/darwin/entrypoint.sh | 57 ----- crc-builder/oci/lib/darwin/run.sh | 136 +++++++++++ crc-builder/oci/lib/linux/entrypoint.sh | 3 - crc-builder/oci/lib/windows/entrypoint.sh | 57 ----- .../windows/{builder/build.ps1 => run.ps1} | 36 +-- .../tkn/tpl/crc-builder-installer.tpl.yaml | 226 ++++++++++-------- 9 files changed, 276 insertions(+), 454 deletions(-) delete mode 100755 crc-builder/oci/lib/common.sh delete mode 100755 crc-builder/oci/lib/darwin/builder/build.sh delete mode 100755 crc-builder/oci/lib/darwin/entrypoint.sh create mode 100755 crc-builder/oci/lib/darwin/run.sh delete mode 100755 crc-builder/oci/lib/windows/entrypoint.sh rename crc-builder/oci/lib/windows/{builder/build.ps1 => run.ps1} (76%) diff --git a/crc-builder/oci/Containerfile.non-linux b/crc-builder/oci/Containerfile.non-linux index a2293a8..ed4f6b7 100644 --- a/crc-builder/oci/Containerfile.non-linux +++ b/crc-builder/oci/Containerfile.non-linux @@ -3,26 +3,23 @@ FROM registry.access.redhat.com/ubi8/ubi-minimal as preparer ARG OS ENV OS ${OS} -COPY lib/${OS}/builder/* /usr/local/crc-builder/ -COPY lib/${OS}/entrypoint.sh /usr/local/bin/entrypoint.sh - -RUN cd /usr/local/crc-builder && \ +RUN cd /tmp && \ if [[ ${OS} == 'windows' ]]; then \ curl -k -LO https://dl.minio.io/client/mc/release/windows-amd64/mc.exe; \ else \ curl -k -LO https://dl.min.io/client/mc/release/darwin-amd64/mc; \ chmod +x mc; \ fi; + +#v0.0.6 +FROM quay.io/rhqp/deliverest@sha256:016f766cac1bb644e1415b1ebb34504394ba0806d52679a55a0ef1162edabb47 -FROM quay.io/rhqp/support-tools:v0.0.2 - -LABEL org.opencontainers.image.authors="CodeReady Containers " +LABEL org.opencontainers.image.authors="CRCQE " -ENV PLATFORM ${PLATFORM} -ENV BUILDER_RESOURCES "/usr/local/crc-builder" +ARG OS -COPY --from=preparer /usr/local/crc-builder ${BUILDER_RESOURCES} -COPY --from=preparer /usr/local/bin/entrypoint.sh /usr/local/bin -COPY lib/common.sh /usr/local/bin +ENV ASSETS_FOLDER=/opt/crc-builder \ + OS=${OS} -ENTRYPOINT entrypoint.sh \ No newline at end of file +COPY --from=preparer /tmp/* ${ASSETS_FOLDER}/ +COPY lib/${OS}/* ${ASSETS_FOLDER}/ \ No newline at end of file diff --git a/crc-builder/oci/lib/common.sh b/crc-builder/oci/lib/common.sh deleted file mode 100755 index d25e713..0000000 --- a/crc-builder/oci/lib/common.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/sh - -# Validate required envs are setup to run the container -validate_envs () { - local validate=1 - - [[ -z "${TARGET_HOST+x}" ]] \ - && echo "TARGET_HOST required" \ - && validate=0 - - [[ -z "${TARGET_HOST_USERNAME+x}" ]] \ - && echo "TARGET_HOST_USERNAME required" \ - && validate=0 - - [[ -z "${TARGET_HOST_KEY_PATH+x}" && -z "${TARGET_HOST_PASSWORD+x}" ]] \ - && echo "TARGET_HOST_KEY_PATH or TARGET_HOST_PASSWORD required" \ - && validate=0 - - return $validate -} - -validate_assets_info () { - local validate=1 - - [[ -z "${TRAY_URL+x}" ]] \ - && echo "TRAY_URL required" \ - && validate=0 - - return $validate -} - -validate_s3_configuration () { - local validate=1 - - [[ -z "${DATALAKE_URL}" || -z "${DATALAKE_ACCESS_KEY}" || -z "${DATALAKE_SECRET_KEY}" ]] \ - && echo "s3 credentials are required, binary can not be updaloaded" \ - && validate=0 - - return $validate -} - -# Define remote connection -remote_connection () { - local remote="${TARGET_HOST_USERNAME}@${TARGET_HOST}" - if [[ ! -z "${TARGET_HOST_DOMAIN+x}" ]]; then - remote="${TARGET_HOST_USERNAME}@${TARGET_HOST_DOMAIN}@${TARGET_HOST}" - fi - echo "${remote}" -} - -# scp connection string -scp_cmd () { - local options='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' - if [[ ! -z "${TARGET_HOST_KEY_PATH+x}" ]]; then - echo "scp -r ${options} -i ${TARGET_HOST_KEY_PATH} " - else - echo "sshpass -p ${TARGET_HOST_PASSWORD} scp -r ${options} " - fi -} - -# ssh connection string -ssh_cmd () { - local options='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' - local connection_string=$(remote_connection) - if [[ ! -z "${TARGET_HOST_KEY_PATH+x}" ]]; then - echo "ssh ${options} -i ${TARGET_HOST_KEY_PATH} ${connection_string}" - else - echo "sshpass -p ${TARGET_HOST_PASSWORD} ssh ${options} ${connection_string}" - fi -} - -upload_path() { - path="distributables/app" - if [[ -z ${CRC_VERSION+x} ]]; then - if [[ ! -z ${PULL_REQUEST+x} ]]; then - echo "${path}/pr-${PULL_REQUEST}" - else - echo "${path}/${REF}" - fi - else - echo "${path}/release/${CRC_VERSION}" - fi -} \ No newline at end of file diff --git a/crc-builder/oci/lib/darwin/builder/build.sh b/crc-builder/oci/lib/darwin/builder/build.sh deleted file mode 100755 index 71df4cd..0000000 --- a/crc-builder/oci/lib/darwin/builder/build.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/bin/sh -# Script to be executed on macos machine to build a crc macos installer -# and upload it to s3 compatible storage - -# Execution is controlled based on ENVS: - -# CUSTOM_BUNDLE_VERSION_VARIABLE When build based on a custom bundle need to set type: PODMAN_VERSION or OPENSHIFT_VERSION -# CUSTOM_BUNDLE_VERSION When build based on a custom bundle need to set version -# CRC_SCM: Source code repository for crc -# CRC_SCM_PR: Optional parameter to build an specific PR for crc -# CRC_SCM_REF: Optional parameter to build an specific PR for crc -# CRC_VERSION: Build based on crc version -# DATALAKE_URL: url for remote s3 compatible storage where build bits will be stored -# DATALAKE_ACCESS_KEY: remote s3 credential -# DATALAKE_SECRET_KEY:remote s3 credential - -# Defaults -CRC_SCM="${CRC_SCM:-"https://github.com/code-ready/crc.git"}" -CRC_SCM_REF="${CRC_SCM_REF:-"main"}" -ADMINHELPER_SCM="${ADMINHELPER_SCM:-"https://github.com/code-ready/admin-helper.git"}" -VFKIT_SCM="${VFKIT_SCM:-"https://github.com/code-ready/vfkit.git"}" - -set -exuo pipefail - -# Upload content to S3 compatible -# $1 remote path -# $2 local path to be uploaded -s3_upload() { - [[ -z "${DATALAKE_URL}" || -z "${DATALAKE_ACCESS_KEY}" || -z "${DATALAKE_SECRET_KEY}" ]] \ - && echo "s3 credentials are required, binary can not be updaloaded" \ - && exit 1 - - ./mc alias set datalake \ - ${DATALAKE_URL} \ - ${DATALAKE_ACCESS_KEY} \ - ${DATALAKE_SECRET_KEY} \ - --api S3v4 - - # Create bucket if not exits - ./mc mb "datalake/${1}" - # Copy files to datalake - ./mc cp "${2}/crc-macos-installer.pkg" "datalake/${1}/crc-macos-installer.pkg" - ./mc cp "${2}/crc-macos-installer.sha256sum" "datalake/${1}/crc-macos-installer.pkg.sha256sum" - # Make bucket public - # ./mc anonymous set public "datalake/${1}/" -} - -##################### -####### MAIN ######## -##################### - -# Custom setup for git -git config --global http.version "HTTP/1.1" -git config --global http.lowSpeedLimit 0 -git config --global http.lowSpeedTime 999999 - -# Get crc code -git clone ${CRC_SCM} - -# Fetch according to parameters provided -if [[ -z ${CRC_VERSION+x} ]]; then - CRC_VERSION_PARTIAL=$(date +'%y.%m.%d') - if [[ ! -z ${CRC_SCM_PR+x} ]]; then - git -C crc fetch origin pull/${CRC_SCM_PR}/head:pr-${CRC_SCM_PR} - git -C crc checkout pr-${CRC_SCM_PR} - else - git -C crc checkout ${CRC_SCM_REF} - fi - # In case we build for a custom bundle we need to match the version - # if [[ ! -z ${CUSTOM_BUNDLE_VERSION_VARIABLE+x} ]] && [[ ! -z ${CUSTOM_BUNDLE_VERSION+x} ]]; then - # sed -i.bak "s/${CUSTOM_BUNDLE_VERSION_VARIABLE} ?= .*/${CUSTOM_BUNDLE_VERSION_VARIABLE} = ${CUSTOM_BUNDLE_VERSION}/g" crc/Makefile - # fi - sed -i.bak "s/CRC_VERSION = .*/CRC_VERSION = ${CRC_VERSION_PARTIAL}/g" crc/Makefile -else - git -C crc checkout "tags/v${CRC_VERSION}" -b "v${CRC_VERSION}" -fi - -# Build admin-helper -git clone ${ADMINHELPER_SCM} -admin_version_line=$(cat admin-helper/crc-admin-helper.spec.in | grep Version:) -admin_version=${admin_version_line##*:} -admin_version=$(echo $admin_version | xargs) -make -C admin-helper out/macos-amd64/crc-admin-helper VERSION=$admin_version - -# Build vfkit -git clone ${VFKIT_SCM} -sudo make -C vfkit all - -# Build pkg -pushd crc -# custom resources to be included -mkdir custom_embedded -cp ./../admin-helper/out/macos-amd64/crc-admin-helper custom_embedded/crc-admin-helper-darwin -cp ./../vfkit/out/vfkit custom_embedded/vfkit -cp ./../vfkit/vf.entitlements custom_embedded/vf.entitlements - -# Match admin-helper version with latest from master head -sed -i '' "s/crcAdminHelperVersion =.*/crcAdminHelperVersion = \"${admin_version}\"/g" pkg/crc/version/version.go - -# create pkg -make out/macos-universal/crc-macos-installer.pkg NO_CODESIGN=1 CUSTOM_EMBED=true EMBED_DOWNLOAD_DIR=custom_embedded -# check sum -pushd out/macos-universal -shasum -a 256 * > crc-macos-installer.sha256sum -popd -popd - -# Upload -s3_upload ${UPLOAD_PATH} crc/out/macos-universal \ No newline at end of file diff --git a/crc-builder/oci/lib/darwin/entrypoint.sh b/crc-builder/oci/lib/darwin/entrypoint.sh deleted file mode 100755 index fc60b4c..0000000 --- a/crc-builder/oci/lib/darwin/entrypoint.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/sh - -source common.sh - -# Envs -if [ "${DEBUG:-}" = "true" ]; then - set -xuo -fi -if [[ ! validate_envs ]] || [[ ! validate_assets_info ]] || [[ ! validate_s3_configuration ]]; then - exit 1 -fi -SSH=$(ssh_cmd) -SCP=$(scp_cmd) - -# Create target on remote -echo "Create builder folder" -target_folder="/Users/${TARGET_HOST_USERNAME}" -$SSH "mkdir -p ${target_folder}" - -# Copy resources -echo "Copy resources to target" -connection_string=$(remote_connection) -$SCP ${BUILDER_RESOURCES} "${connection_string}:${target_folder}" - -# Run builder -echo "Running builder" -build_cmd="DATALAKE_URL=${DATALAKE_URL} DATALAKE_ACCESS_KEY=${DATALAKE_ACCESS_KEY} DATALAKE_SECRET_KEY=${DATALAKE_SECRET_KEY} " -if [[ ! -z ${CRC_SCM+x} ]]; then - build_cmd="${build_cmd} CRC_SCM=${CRC_SCM} " -fi -if [[ -z ${CRC_VERSION+x} ]]; then - if [[ ! -z ${CUSTOM_BUNDLE_VERSION_VARIABLE+x} ]]; then - build_cmd="${build_cmd} CUSTOM_BUNDLE_VERSION_VARIABLE=${CUSTOM_BUNDLE_VERSION_VARIABLE} " - fi - if [[ ! -z ${CUSTOM_BUNDLE_VERSION+x} ]]; then - build_cmd="${build_cmd} CUSTOM_BUNDLE_VERSION=${CUSTOM_BUNDLE_VERSION} " - fi - if [[ ! -z ${PULL_REQUEST+x} ]]; then - build_cmd="${build_cmd} CRC_SCM_PR=${PULL_REQUEST} " - fi - if [[ ! -z ${REF+x} ]]; then - build_cmd="${build_cmd} CRC_SCM_REF=${REF} " - fi -else - build_cmd="${build_cmd} CRC_VERSION=${CRC_VERSION} " -fi -# UPLOAD PATH, create it as local env and then pass to remote execution -# creating it as local we can pick the value from the task -UPLOAD_PATH="${UPLOAD_PATH:-"$(upload_path)"}" -build_cmd="${build_cmd} UPLOAD_PATH=${UPLOAD_PATH} " - -build_cmd="${build_cmd} ./build.sh" -$SSH "cd ${target_folder}/crc-builder && DEBUG=true ${build_cmd}" - -# Cleanup -echo "Cleanup target" -$SSH "sudo rm -fr ${target_folder}/crc-builder" \ No newline at end of file diff --git a/crc-builder/oci/lib/darwin/run.sh b/crc-builder/oci/lib/darwin/run.sh new file mode 100755 index 0000000..e72014a --- /dev/null +++ b/crc-builder/oci/lib/darwin/run.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +# Parameters +crcSCM="https://github.com/code-ready/crc.git" +crcSCMPR='' +crcSCMRef='main' +uploadPath='crc-binaries' +datalakeURL='' +datalakeAcessKey='' +datalakeSecretKey='' + +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -crcSCM) + crcSCM="$2" + shift + shift + ;; + -crcSCMPR) + crcSCMPR="$2" + shift + shift + ;; + -crcSCMRef) + crcSCMRef="$2" + shift + shift + ;; + -uploadPath) + uploadPath="$2" + shift + shift + ;; + -datalakeURL) + datalakeURL="$2" + shift + shift + ;; + -datalakeAcessKey) + datalakeAcessKey="$2" + shift + shift + ;; + -datalakeSecretKey) + datalakeSecretKey="$2" + shift + shift + ;; + *) # unknown option + shift + ;; + esac +done + +set -exuo pipefail + +# Upload content to S3 compatible +# $1 remote path +# $2 local path to be uploaded +s3_upload() { + [[ -z "$datalakeURL" || -z "$datalakeAcessKey" || -z "$datalakeSecretKey" ]] \ + && echo "s3 credentials are required, binary can not be updaloaded" \ + && exit 1 + + ./mc alias set datalake \ + $datalakeURL \ + $datalakeAcessKey \ + $datalakeSecretKey \ + --api S3v4 + + # Create bucket if not exits + ./mc mb "datalake/${1}" + # Copy files to datalake + ./mc cp "${2}/crc-macos-installer.pkg" "datalake/${1}/crc-macos-installer.pkg" + ./mc cp "${2}/crc-macos-installer.sha256sum" "datalake/${1}/crc-macos-installer.pkg.sha256sum" + # Make bucket public + # ./mc anonymous set public "datalake/${1}/" +} + +##################### +####### MAIN ######## +##################### + +# Custom setup for git +git config --global http.version "HTTP/1.1" +git config --global http.lowSpeedLimit 0 +git config --global http.lowSpeedTime 999999 + +# Get crc code +git clone ${CRC_SCM} + +# Fetch according to parameters provided +CRC_VERSION_PARTIAL=$(date +'%y.%m.%d') +if [ ! -z "$crcSCMPR" ] +then + git -C crc fetch origin pull/$crcSCMPR/head:pr-$crcSCMPR + git -C crc checkout pr-$crcSCMPR +else + git -C crc checkout $crcSCMRef +fi +sed -i.bak "s/CRC_VERSION = .*/CRC_VERSION = ${CRC_VERSION_PARTIAL}/g" crc/Makefile + + +# Build admin-helper +git clone https://github.com/code-ready/admin-helper.git +admin_version_line=$(cat admin-helper/crc-admin-helper.spec.in | grep Version:) +admin_version=${admin_version_line##*:} +admin_version=$(echo $admin_version | xargs) +make -C admin-helper macos-universal VERSION=$admin_version + +# Build vfkit +git clone https://github.com/code-ready/vfkit.git +sudo make -C vfkit all + +# Build pkg +pushd crc +# custom resources to be included +mkdir custom_embedded +cp ./../admin-helper/out/macos-universal/crc-admin-helper custom_embedded/crc-admin-helper-darwin +cp ./../vfkit/out/vfkit custom_embedded/vfkit +cp ./../vfkit/vf.entitlements custom_embedded/vf.entitlements + +# Match admin-helper version with latest from master head +sed -i '' "s/crcAdminHelperVersion =.*/crcAdminHelperVersion = \"${admin_version}\"/g" pkg/crc/version/version.go + +# create pkg +make out/macos-universal/crc-macos-installer.pkg NO_CODESIGN=1 CUSTOM_EMBED=true EMBED_DOWNLOAD_DIR=custom_embedded +# check sum +pushd out/macos-universal +shasum -a 256 * > crc-macos-installer.sha256sum +popd +popd + +# Upload +s3_upload $uploadPath crc/out/macos-universal \ No newline at end of file diff --git a/crc-builder/oci/lib/linux/entrypoint.sh b/crc-builder/oci/lib/linux/entrypoint.sh index 6040319..2a2779a 100755 --- a/crc-builder/oci/lib/linux/entrypoint.sh +++ b/crc-builder/oci/lib/linux/entrypoint.sh @@ -1,8 +1,5 @@ #!/bin/bash -# Imports -source ./common.sh - # Script to be executed on macos machine to build a crc macos installer # and upload it to s3 compatible storage diff --git a/crc-builder/oci/lib/windows/entrypoint.sh b/crc-builder/oci/lib/windows/entrypoint.sh deleted file mode 100755 index 0637de7..0000000 --- a/crc-builder/oci/lib/windows/entrypoint.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/sh - -source common.sh - -# Envs -if [ "${DEBUG:-}" = "true" ]; then - set -xuo -fi -if [[ ! validate_envs ]] || [[ ! validate_assets_info ]] || [[ ! validate_s3_configuration ]]; then - exit 1 -fi -SSH=$(ssh_cmd) -SCP=$(scp_cmd) - -# Create target on remote -echo "Create builder folder" -target_folder="/Users/${TARGET_HOST_USERNAME}" -$SSH "powershell.exe -c New-Item -ItemType directory -Path ${target_folder}" - -# Copy resources -echo "Copy resources to target" -connection_string=$(remote_connection) -$SCP ${BUILDER_RESOURCES} "${connection_string}:${target_folder}" - -# Run builder -echo "Running builder" -build_cmd=".\build.ps1 -datalakeURL ${DATALAKE_URL} -datalakeAcessKey ${DATALAKE_ACCESS_KEY} -datalakeSecretKey ${DATALAKE_SECRET_KEY} " -if [[ ! -z ${CRC_SCM+x} ]]; then - build_cmd="${build_cmd} -crcSCM ${CRC_SCM} " -fi -if [[ -z ${CRC_VERSION+x} ]]; then - if [[ ! -z ${CUSTOM_BUNDLE_VERSION_VARIABLE+x} ]]; then - build_cmd="${build_cmd} -customBundleVersionVariable ${CUSTOM_BUNDLE_VERSION_VARIABLE} " - fi - if [[ ! -z ${CUSTOM_BUNDLE_VERSION+x} ]]; then - build_cmd="${build_cmd} -customBundleVersion ${CUSTOM_BUNDLE_VERSION} " - fi - if [[ ! -z ${PULL_REQUEST+x} ]]; then - build_cmd="${build_cmd} -crcSCMPR ${PULL_REQUEST} " - fi - if [[ ! -z ${REF+x} ]]; then - build_cmd="${build_cmd} -crcSCMRef ${REF} " - fi -else - build_cmd="${build_cmd} -crcVersion ${CRC_VERSION} " -fi - -# UPLOAD PATH, create it as local env and then pass to remote execution -# creating it as local we can pick the value from the task -UPLOAD_PATH="${UPLOAD_PATH:-"$(upload_path)"}" -build_cmd="${build_cmd} -uploadPath ${UPLOAD_PATH} " - -$SSH "cd ${target_folder}/crc-builder; ${build_cmd}" - -# Cleanup -echo "Cleanup target" -$SSH "rm -r ${target_folder}/crc-builder -Force" \ No newline at end of file diff --git a/crc-builder/oci/lib/windows/builder/build.ps1 b/crc-builder/oci/lib/windows/run.ps1 similarity index 76% rename from crc-builder/oci/lib/windows/builder/build.ps1 rename to crc-builder/oci/lib/windows/run.ps1 index b2dceb4..dd11e85 100644 --- a/crc-builder/oci/lib/windows/builder/build.ps1 +++ b/crc-builder/oci/lib/windows/run.ps1 @@ -1,17 +1,10 @@ # Script to be executed on windows machine to build a crc windows installer # and upload it to s3 compatible storage param( - [Parameter(HelpMessage='When build based on a custom bundle need to set type: PODMAN_VERSION or OPENSHIFT_VERSION')] - $customBundleVersionVariable, - [Parameter(HelpMessage='When build based on a custom bundle need to set version')] - $customBundleVersion, [Parameter(HelpMessage='crc scm')] $crcSCM="https://github.com/code-ready/crc.git", [Parameter(HelpMessage='Optional parameter to build an specific PR for crc')] $crcSCMPR, - # Review this one - [Parameter(HelpMessage='Optinal crc version to build an specfic')] - $crcVersion, [Parameter(HelpMessage='crc scm ref')] $crcSCMRef="main", [Parameter(HelpMessage='upload path on remote storage where upload the artifacts')] @@ -70,27 +63,16 @@ git clone $crcSCM pushd crc # Fetch according to parameters provided -if (! $PSBoundParameters.ContainsKey('crcVersion')) { - $crcVersionPartial=Get-Date -format "yy.MM.dd" - if ($PSBoundParameters.ContainsKey('crcSCMPR')) { - git fetch origin pull/$crcSCMPR/head:pr-$crcSCMPR - git checkout pr-$crcSCMPR - } else { - git checkout $crcSCMRef - } - # In case we build for a custom bundle we need to match the version - # if ($PSBoundParameters.ContainsKey('customBundleVersionVariable') -And $PSBoundParameters.ContainsKey('customBundleVersion')) { - # (Get-Content -path Makefile) ` - # -replace "$customBundleVersionVariable \?= .*","$customBundleVersionVariable ?= $customBundleVersion" ` - # | Set-Content -path Makefile - # } - (Get-Content -path Makefile) ` - -replace 'CRC_VERSION = .*',"CRC_VERSION = $crcVersionPartial" ` - | Set-Content -path Makefile -} -else { - git checkout "v$crcVersion" +$crcVersionPartial=Get-Date -format "yy.MM.dd" +if ($PSBoundParameters.ContainsKey('crcSCMPR')) { + git fetch origin pull/$crcSCMPR/head:pr-$crcSCMPR + git checkout pr-$crcSCMPR +} else { + git checkout $crcSCMRef } +(Get-Content -path Makefile) ` + -replace 'CRC_VERSION = .*',"CRC_VERSION = $crcVersionPartial" ` + | Set-Content -path Makefile popd # Build admin-helper diff --git a/crc-builder/tkn/tpl/crc-builder-installer.tpl.yaml b/crc-builder/tkn/tpl/crc-builder-installer.tpl.yaml index ef1c719..70665ff 100644 --- a/crc-builder/tkn/tpl/crc-builder-installer.tpl.yaml +++ b/crc-builder/tkn/tpl/crc-builder-installer.tpl.yaml @@ -18,117 +18,133 @@ spec: description: >- This task will build openshift local installers + workspaces: + - name: s3-credentials + description: | + ocp secret holding the s3 credentials. Secret should be accessible to this task. + --- + apiVersion: v1 + kind: Secret + metadata: + name: XXXX + labels: + app.kubernetes.io/component: XXXX + type: Opaque + data: + download-url: ${download_url} + upload-url: ${upload_url} + bucket: ${bucket_value} + access-key: ${access_key} + secret-key: ${secret_key} + mountPath: /opt/s3-credentials + - name: host-info + description: | + ocp secret holding the hsot info credentials. Secret should be accessible to this task. + --- + apiVersion: v1 + kind: Secret + metadata: + name: XXXX + labels: + app.kubernetes.io/component: XXXX + type: Opaque + data: + host: XXXX + user: XXXX + password: XXXX + key: XXXX + platform: XXXX + os-version: XXXX + arch: XXXX + os: XXXX + mountPath: /opt/host/ + params: - - name: crc-scm - default: "''" - - name: crc-scm-pr - default: "''" - - name: crc-scm-ref - default: main - - name: custom-bundle-version-variable - description: | - When building an installer based on a custom bundle it is required - to manipulate the Makefile to match the custom version. - Current variables holding bundle version based on bundle type are - * OPENSHIFT_VERSION - * PODMAN_VERSION - default: "''" - - name: custom-bundle-version - default: "''" - - name: os - description: valid values are macos and windows - - name: host-config-secret - description: secret holding a host config - - name: s3-url - - name: s3-access-key - - name: s3-secret-key - - name: s3-folder-path - - name: s3-download-url + # Manage task params + - name: os + description: valid values are macos and windows + # SCM params + - name: crc-scm + default: https://github.com/code-ready/crc.git + - name: crc-scm-ref + default: main + - name: crc-scm-pr + default: "''" + # Control params + - name: debug + description: debug purposes extend verbosity on cmds executed on the target + default: 'false' results: - - name: downloadable-base-url - description: base url where the installer and the shasumfile can be downloaded - - name: distributable-name - description: distributable file name for the installer - - name: shasumfile - description: shasumfile name + - name: downloadable-base-url + description: base url where the installer and the shasumfile can be downloaded + - name: distributable-name + description: distributable file name for the installer + - name: shasumfile + description: shasumfile name - volumes: - - name: host-connection - secret: - secretName: $(params.host-config-secret) - steps: - - name: crc-executable-builder - # cimage and cversion values should be passed to the template - image: cimage:cversion-$(params.os) - imagePullPolicy: Always - volumeMounts: - - mountPath: /opt/host/ - name: host-connection - script: | - #!/bin/sh - - # Copy key to connect to machine - cp /opt/host/key id_rsa - chmod 600 id_rsa - - # Run builder on target machine - TARGET_HOST=$(cat /opt/host/host) - TARGET_HOST_USERNAME=$(cat /opt/host/user) - TARGET_HOST_KEY_PATH=id_rsa - # sanitizing permissions for the key - chmod 600 ${TARGET_HOST_KEY_PATH} - - DATALAKE_URL=$(params.s3-url) - DATALAKE_ACCESS_KEY=$(params.s3-access-key) - DATALAKE_SECRET_KEY=$(params.s3-secret-key) - UPLOAD_PATH=$(params.s3-folder-path) + - name: crc-executable-builder + # cimage and cversion values should be passed to the template + image: cimage:cversion-$(params.os) + imagePullPolicy: Always + script: | + #!/bin/sh - # Optionals - if [[ $(params.crc-scm) != "" ]]; then - CRC_SCM=$(params.crc-scm) - fi - if [[ $(params.crc-scm-pr) != "" ]]; then - PULL_REQUEST=$(params.crc-scm-pr) - fi - if [[ $(params.crc-scm-ref) != "" ]]; then - REF=$(params.crc-scm-ref) - fi - if [[ $(params.custom-bundle-version-variable) != "" ]]; then - CUSTOM_BUNDLE_VERSION_VARIABLE=$(params.custom-bundle-version-variable) - fi - if [[ $(params.custom-bundle-version) != "" ]]; then - CUSTOM_BUNDLE_VERSION=$(params.custom-bundle-version) - fi - - # set -exuo pipefail + # Prepare ENVs + SECONDS=0 + DEBUG=$(params.debug) + TARGET_HOST=$(cat /opt/host/host) + TARGET_HOST_USERNAME=$(cat /opt/host/user) + cp /opt/host/key id_rsa + chmod 600 id_rsa + TARGET_HOST_KEY_PATH=id_rsa + TARGET_FOLDER=crc-builder + TARGET_CLEANUP='true' + + # Create cmd per OS + runner="run.sh" + if [[ $(params.os) == "windows" ]]; then + runner="run.ps1" + fi + cmd="${TARGET_FOLDER}/${runner} -crcSCM $(params.crc-scm) " + cmd+="-crcSCMRef $(params.crc-scm-ref) " + if [[ $(params.crc-scm-pr) != "" ]]; then + cmd+="-crcSCMPR $(params.crc-scm-pr) " + fi + cmd+="-uploadPath $(cat /opt/s3-credentials/bucket)/$(params.s3-folder-path) " + cmd+="-datalakeURL $(cat /opt/s3-credentials/upload-url) " + cmd+="-datalakeAcessKey $(cat /opt/s3-credentials/access-key) " + cmd+="-datalakeSecretKey $(cat /opt/s3-credentials/secret-key) " - # Build installer - . entrypoint.sh + # Exec + . entrypoint.sh "${cmd}" + if [[ $? -ne 0 ]]; then + exit 1 + fi - echo -n "$(params.s3-download-url)" \ - | tee $(results.downloadable-base-url.path) - case "$(params.os)" in - macos|darwin) - echo -n "crc-macos-installer.pkg" | tee $(results.distributable-name.path) - echo -n "crc-macos-installer.pkg.sha256sum" | tee $(results.shasumfile.path) - ;; - windows) - echo -n "crc-windows-installer.zip" | tee $(results.distributable-name.path) - echo -n "crc-windows-installer.zip.sha256sum" | tee $(results.shasumfile.path) - ;; - *) - echo -n "" | tee $(results.distributable-name.path) - echo -n "" | tee $(results.shasumfile.path) - ;; - esac + echo -n "$(params.s3-download-url)" \ + | tee $(results.downloadable-base-url.path) + case "$(params.os)" in + macos|darwin) + echo -n "crc-macos-installer.pkg" | tee $(results.distributable-name.path) + echo -n "crc-macos-installer.pkg.sha256sum" | tee $(results.shasumfile.path) + ;; + windows) + echo -n "crc-windows-installer.zip" | tee $(results.distributable-name.path) + echo -n "crc-windows-installer.zip.sha256sum" | tee $(results.shasumfile.path) + ;; + *) + echo -n "" | tee $(results.distributable-name.path) + echo -n "" | tee $(results.shasumfile.path) + ;; + esac - resources: - requests: - memory: 150Mi - cpu: 90m - limits: - memory: 270Mi - cpu: 150m - timeout: 90m \ No newline at end of file + resources: + requests: + memory: 150Mi + cpu: 90m + limits: + memory: 270Mi + cpu: 150m + timeout: 90m \ No newline at end of file From 4c85ff0ef0caf21624ef8822a436c79297d7f9af Mon Sep 17 00:00:00 2001 From: Adrian Riobo Date: Tue, 22 Oct 2024 13:12:20 +0200 Subject: [PATCH 2/2] chore: sync crc-builder linux amd64 tekton task contract. Fix #37. Signed-off-by: Adrian Riobo --- crc-builder/oci/Containerfile.linux | 1 - crc-builder/oci/lib/darwin/run.sh | 12 +- crc-builder/oci/lib/windows/run.ps1 | 4 + .../tkn/tpl/crc-builder-arm64.tpl.yaml | 22 ++- .../tkn/tpl/crc-builder-installer.tpl.yaml | 6 +- crc-builder/tkn/tpl/crc-builder.tpl.yaml | 129 ++++++++++-------- 6 files changed, 100 insertions(+), 74 deletions(-) diff --git a/crc-builder/oci/Containerfile.linux b/crc-builder/oci/Containerfile.linux index 4dcc814..fa18b9b 100644 --- a/crc-builder/oci/Containerfile.linux +++ b/crc-builder/oci/Containerfile.linux @@ -15,7 +15,6 @@ RUN microdnf -y install git make gcc libvirt-devel perl-Digest-SHA xz findutils && chmod +x /usr/local/bin/mc \ && rm /tmp/${GO_VERSION}.tar.gz -COPY lib/common.sh /usr/local/bin COPY lib/linux/entrypoint.sh /usr/local/bin/entrypoint.sh ENTRYPOINT entrypoint.sh \ No newline at end of file diff --git a/crc-builder/oci/lib/darwin/run.sh b/crc-builder/oci/lib/darwin/run.sh index e72014a..98380df 100755 --- a/crc-builder/oci/lib/darwin/run.sh +++ b/crc-builder/oci/lib/darwin/run.sh @@ -4,6 +4,7 @@ crcSCM="https://github.com/code-ready/crc.git" crcSCMPR='' crcSCMRef='main' +targetFolder="crc-builder" uploadPath='crc-binaries' datalakeURL='' datalakeAcessKey='' @@ -27,6 +28,11 @@ while [[ $# -gt 0 ]]; do shift shift ;; + -targetFolder) + targetFolder="$2" + shift + shift + ;; -uploadPath) uploadPath="$2" shift @@ -82,13 +88,15 @@ s3_upload() { ####### MAIN ######## ##################### +cd $targetFolder + # Custom setup for git git config --global http.version "HTTP/1.1" git config --global http.lowSpeedLimit 0 git config --global http.lowSpeedTime 999999 # Get crc code -git clone ${CRC_SCM} +git clone $crcSCM # Fetch according to parameters provided CRC_VERSION_PARTIAL=$(date +'%y.%m.%d') @@ -111,7 +119,7 @@ make -C admin-helper macos-universal VERSION=$admin_version # Build vfkit git clone https://github.com/code-ready/vfkit.git -sudo make -C vfkit all +make -C vfkit all # Build pkg pushd crc diff --git a/crc-builder/oci/lib/windows/run.ps1 b/crc-builder/oci/lib/windows/run.ps1 index dd11e85..fc18020 100644 --- a/crc-builder/oci/lib/windows/run.ps1 +++ b/crc-builder/oci/lib/windows/run.ps1 @@ -7,6 +7,8 @@ param( $crcSCMPR, [Parameter(HelpMessage='crc scm ref')] $crcSCMRef="main", + [Parameter(HelpMessage='folder on the remote target to move all assets to run the builder')] + $targetFolder="crc-builder", [Parameter(HelpMessage='upload path on remote storage where upload the artifacts')] $uploadPath, [Parameter(Mandatory,HelpMessage='url for remote s3 compatible storage where build bits will be stored')] @@ -53,6 +55,8 @@ function Get-UploadPath($crcVersion, $crcSCMPR, $crcSCMRef) { ####### MAIN ########## ####################### +cd $targetFolder + # Custom setup for git git config --global http.version "HTTP/1.1" git config --global http.lowSpeedLimit 0 diff --git a/crc-builder/tkn/tpl/crc-builder-arm64.tpl.yaml b/crc-builder/tkn/tpl/crc-builder-arm64.tpl.yaml index 47c2d1e..13d26a5 100644 --- a/crc-builder/tkn/tpl/crc-builder-arm64.tpl.yaml +++ b/crc-builder/tkn/tpl/crc-builder-arm64.tpl.yaml @@ -58,13 +58,15 @@ spec: # SCM params - name: crc-scm default: https://github.com/code-ready/crc.git - - name: crc-scm-pr - default: "''" - name: crc-scm-ref default: main + - name: crc-scm-pr + default: "''" + # Target params - name: s3-folder-path + default: 'crc-binaries' # Builder params - name: builder-cpus @@ -140,19 +142,15 @@ spec: } cmd="podman run --rm --name crc-builder -d " - cmd+="-e DATALAKE_URL=$(cat /opt/s3-credentials/upload-url) " - cmd+="-e DATALAKE_ACCESS_KEY=$(cat /opt/s3-credentials/access-key) " - cmd+="-e DATALAKE_SECRET_KEY=$(cat /opt/s3-credentials/secret-key) " - # Optionals - if [[ $(params.crc-scm) != "" ]]; then - cmd+="-e CRC_SCM=$(params.crc-scm) " - fi + # SCM + cmd+="-e CRC_SCM=$(params.crc-scm) " + cmd+="-e CRC_SCM_REF=$(params.crc-scm-ref) " if [[ $(params.crc-scm-pr) != "" ]]; then cmd+="-e CRC_SCM_PR=$(params.crc-scm-pr) " fi - if [[ $(params.crc-scm-ref) != "" ]]; then - cmd+="-e CRC_SCM_REF=$(params.crc-scm-ref) " - fi + cmd+="-e DATALAKE_URL=$(cat /opt/s3-credentials/upload-url) " + cmd+="-e DATALAKE_ACCESS_KEY=$(cat /opt/s3-credentials/access-key) " + cmd+="-e DATALAKE_SECRET_KEY=$(cat /opt/s3-credentials/secret-key) " if [[ $(params.s3-folder-path) != "" ]]; then cmd+="-e UPLOAD_PATH=$(cat /opt/s3-credentials/bucket)/$(params.s3-folder-path) " fi diff --git a/crc-builder/tkn/tpl/crc-builder-installer.tpl.yaml b/crc-builder/tkn/tpl/crc-builder-installer.tpl.yaml index 70665ff..98457c2 100644 --- a/crc-builder/tkn/tpl/crc-builder-installer.tpl.yaml +++ b/crc-builder/tkn/tpl/crc-builder-installer.tpl.yaml @@ -70,6 +70,9 @@ spec: default: main - name: crc-scm-pr default: "''" + # Target params + - name: s3-folder-path + default: 'crc-binaries' # Control params - name: debug description: debug purposes extend verbosity on cmds executed on the target @@ -112,6 +115,7 @@ spec: if [[ $(params.crc-scm-pr) != "" ]]; then cmd+="-crcSCMPR $(params.crc-scm-pr) " fi + cmd+="-targetFolder ${TARGET_FOLDER} " cmd+="-uploadPath $(cat /opt/s3-credentials/bucket)/$(params.s3-folder-path) " cmd+="-datalakeURL $(cat /opt/s3-credentials/upload-url) " cmd+="-datalakeAcessKey $(cat /opt/s3-credentials/access-key) " @@ -123,7 +127,7 @@ spec: exit 1 fi - echo -n "$(params.s3-download-url)" \ + echo -n "$(cat /opt/s3-credentials/download-url)/$(params.s3-folder-path)" \ | tee $(results.downloadable-base-url.path) case "$(params.os)" in macos|darwin) diff --git a/crc-builder/tkn/tpl/crc-builder.tpl.yaml b/crc-builder/tkn/tpl/crc-builder.tpl.yaml index 919afe6..ee3ac04 100644 --- a/crc-builder/tkn/tpl/crc-builder.tpl.yaml +++ b/crc-builder/tkn/tpl/crc-builder.tpl.yaml @@ -18,70 +18,83 @@ spec: description: >- This task will build openshift local binary for linux distributions + workspaces: + - name: s3-credentials + description: | + ocp secret holding the s3 credentials. Secret should be accessible to this task. + --- + apiVersion: v1 + kind: Secret + metadata: + name: XXXX + labels: + app.kubernetes.io/component: XXXX + type: Opaque + data: + download-url: ${download_url} + upload-url: ${upload_url} + bucket: ${bucket_value} + access-key: ${access_key} + secret-key: ${secret_key} + mountPath: /opt/s3-credentials + params: - - name: crc-scm - default: https://github.com/code-ready/crc.git - - name: crc-scm-pr - default: "''" - - name: crc-scm-ref - default: main - - name: s3-url - - name: s3-access-key - - name: s3-secret-key - - name: s3-folder-path - - name: s3-download-url + # SCM params + - name: crc-scm + default: https://github.com/code-ready/crc.git + - name: crc-scm-ref + default: main + - name: crc-scm-pr + default: "''" + # Target params + - name: s3-folder-path + default: 'crc-binaries' + results: - - name: downloadable-base-url - description: base url where the installer and the shasumfile can be downloaded - - name: distributable-name - description: distributable file name for the installer - - name: shasumfile - description: shasumfile name + - name: downloadable-base-url + description: base url where the installer and the shasumfile can be downloaded + - name: distributable-name + description: distributable file name for the installer + - name: shasumfile + description: shasumfile name steps: - - name: build - # cimage and cversion values should be passed to the template - image: cimage:cversion-linux - imagePullPolicy: Always - script: | - #!/bin/sh - DATALAKE_URL=$(params.s3-url) - DATALAKE_ACCESS_KEY=$(params.s3-access-key) - DATALAKE_SECRET_KEY=$(params.s3-secret-key) + - name: build + # cimage and cversion values should be passed to the template + image: cimage:cversion-linux + imagePullPolicy: Always + script: | + #!/bin/sh - # Optionals - if [[ $(params.crc-scm) != "" ]]; then + # SCM CRC_SCM=$(params.crc-scm) - fi - if [[ $(params.crc-scm-pr) != "" ]]; then - CRC_SCM_PR=$(params.crc-scm-pr) - fi - if [[ $(params.crc-scm-ref) != "" ]]; then CRC_SCM_REF=$(params.crc-scm-ref) - fi - if [[ $(params.s3-folder-path) != "" ]]; then - UPLOAD_PATH=$(params.s3-folder-path) - fi - - # Build installer - DEBUG=true - . entrypoint.sh - - if [[ $? -ne 0 ]]; then - exit 1 - fi + if [[ $(params.crc-scm-pr) != "" ]]; then + CRC_SCM_PR=$(params.crc-scm-pr) + fi + DATALAKE_URL=$(cat /opt/s3-credentials/upload-url) + DATALAKE_ACCESS_KEY=$(cat /opt/s3-credentials/access-key) + DATALAKE_SECRET_KEY=$(cat /opt/s3-credentials/secret-key) + UPLOAD_PATH=$(cat /opt/s3-credentials/bucket)/$(params.s3-folder-path) + + # Build installer + DEBUG=true + . entrypoint.sh + if [[ $? -ne 0 ]]; then + exit 1 + fi - # From entrypoint we can get UPLOAD_PATH env with the target bucket - echo -n "$(params.s3-download-url)/${UPLOAD_PATH}" | tee $(results.downloadable-base-url.path) - # Linux generated files - echo -n "crc-linux-amd64.tar.xz" | tee $(results.distributable-name.path) - echo -n "crc-linux-amd64.tar.xz.sha256sum" | tee $(results.shasumfile.path) - resources: - requests: - memory: 450Mi - cpu: 250m - limits: - memory: 3800Mi - cpu: 1850m - timeout: 900m \ No newline at end of file + # From entrypoint we can get UPLOAD_PATH env with the target bucket + echo -n "$(cat /opt/s3-credentials/download-url)/$(params.s3-folder-path)" | tee $(results.downloadable-base-url.path) + # Linux generated files + echo -n "crc-linux-amd64.tar.xz" | tee $(results.distributable-name.path) + echo -n "crc-linux-amd64.tar.xz.sha256sum" | tee $(results.shasumfile.path) + resources: + requests: + memory: 450Mi + cpu: 250m + limits: + memory: 3800Mi + cpu: 1850m + timeout: 900m \ No newline at end of file