From dca632c7856697c75075552d9509b0f2a2eb9924 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 28 Jun 2024 12:53:04 +0000 Subject: [PATCH] Build & release arm64 (#42) * Use reusable workflows * arm64 * Remove override-stage Current `override-stage` has no effect on final rock I inspected the rock we're currently building (i.e. ghcr.io/canonical/charmed-mysql:8.0.37-22.04_edge), and it looks like it doesn't contain `/var/lib/mysql` ``` root@foo-0:/# find / -name mysql /home/mysql /usr/lib/x86_64-linux-gnu/perl5/5.34/DBD/mysql /usr/lib/x86_64-linux-gnu/perl5/5.34/auto/DBD/mysql /usr/lib/mysql /usr/share/mysql /usr/bin/mysql /var/log/mysql /etc/init.d/mysql /etc/mysql root@foo-0:/# mysqld --initialize-insecure -u mysql root@foo-0:/# find / -name mysql /home/mysql /usr/lib/x86_64-linux-gnu/perl5/5.34/DBD/mysql /usr/lib/x86_64-linux-gnu/perl5/5.34/auto/DBD/mysql /usr/lib/mysql /usr/share/mysql /usr/bin/mysql /var/log/mysql /var/lib/mysql /var/lib/mysql/mysql /etc/init.d/mysql /etc/mysql ``` so I think the entire `override-stage` in our rock is doing nothing --- .github/workflows/build.yaml | 37 -------- .../workflows/{check_version.yaml => ci.yaml} | 30 +++++- .github/workflows/integration.yaml | 41 --------- .github/workflows/lint.yaml | 16 ---- .github/workflows/publish.yaml | 50 ---------- .github/workflows/release.yaml | 62 +++++++++++++ .github/workflows/sbom.yaml | 31 ------- .github/workflows/sync_issue_to_jira.yaml | 2 +- README.md | 3 +- rockcraft.yaml | 37 ++------ tox.ini | 91 +------------------ 11 files changed, 99 insertions(+), 301 deletions(-) delete mode 100644 .github/workflows/build.yaml rename .github/workflows/{check_version.yaml => ci.yaml} (54%) delete mode 100644 .github/workflows/integration.yaml delete mode 100644 .github/workflows/lint.yaml delete mode 100644 .github/workflows/publish.yaml create mode 100644 .github/workflows/release.yaml delete mode 100644 .github/workflows/sbom.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml deleted file mode 100644 index 4408fba..0000000 --- a/.github/workflows/build.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: Build ROCK - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - workflow_call: - -jobs: - build: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Setup LXD - uses: canonical/setup-lxd@main - - name: Install dependencies - run: | - sudo snap install yq - sudo snap install --classic --channel edge rockcraft - - name: Build ROCK - id: pack - run: rockcraft pack - - name: Upload rockcraft logs - if: ${{ failure() && steps.pack.outcome == 'failure' }} - uses: actions/upload-artifact@v3 - with: - name: rockcraft-build-logs - path: ~/.local/state/rockcraft/log/ - if-no-files-found: error - - name: Upload locally built ROCK artifact - uses: actions/upload-artifact@v3 - with: - name: charmed-mysql-rock - path: "charmed-mysql_*.rock" diff --git a/.github/workflows/check_version.yaml b/.github/workflows/ci.yaml similarity index 54% rename from .github/workflows/check_version.yaml rename to .github/workflows/ci.yaml index 96fd77d..1c16f80 100644 --- a/.github/workflows/check_version.yaml +++ b/.github/workflows/ci.yaml @@ -1,14 +1,34 @@ -name: Check Version +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. +name: Tests + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + on: - workflow_dispatch: pull_request: jobs: + lint: + name: Lint + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install tox + run: pipx install tox + - name: Run linters + run: tox run -e lint + check-version: + name: Check version runs-on: ubuntu-22.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@v4 - name: Install yq run: sudo snap install yq - name: Compare versions @@ -25,3 +45,7 @@ jobs: echo "SNAP version: $SNAP_VERSION" exit 1 fi + + build: + name: Build rock + uses: canonical/data-platform-workflows/.github/workflows/build_rock.yaml@v16 diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml deleted file mode 100644 index ac40a4f..0000000 --- a/.github/workflows/integration.yaml +++ /dev/null @@ -1,41 +0,0 @@ -name: Operator Tests - -on: - pull_request: - workflow_call: - workflow_dispatch: - -jobs: - build: - uses: ./.github/workflows/build.yaml - integration: - runs-on: ubuntu-latest - needs: build - strategy: - matrix: - env: - - integration-charm - - integration-database-relation - - integration-osm-mysql - - integration-replication - - integration-self-healing - - integration-tls - - integration-backups - fail-fast: false - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main - with: - provider: microk8s - # This is needed until - # https://bugs.launchpad.net/juju/+bug/1977582 is fixed - bootstrap-options: "--agent-version 2.9.29" - - uses: actions/download-artifact@v3 - with: - name: charmed-mysql-rock - - name: Install tox - run: python3 -m pip install tox - - name: Integration Tests - run: sg microk8s -c "tox -e ${{ matrix.env }}" diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml deleted file mode 100644 index 449ffe6..0000000 --- a/.github/workflows/lint.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: Lint -on: - pull_request: - workflow_call: - workflow_dispatch: - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Install tox - run: python3 -m pip install tox - - name: YAML Lint - run: tox -e lint diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml deleted file mode 100644 index 6a2837b..0000000 --- a/.github/workflows/publish.yaml +++ /dev/null @@ -1,50 +0,0 @@ -name: Publish ROCK -on: - push: - branches: - - 8.0-22.04 - workflow_dispatch: - branches: - - 8.0-22.04 - -jobs: - sbom: - uses: ./.github/workflows/sbom.yaml - publish: - needs: sbom - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Setup Docker - run: | - sudo snap install docker - sudo addgroup --system docker; sudo adduser $USER docker - newgrp docker - sudo snap disable docker; sudo snap enable docker - - name: Install skopeo - run: | - sudo snap install --devmode --channel edge skopeo - - name: Install yq - run: | - sudo snap install yq - - uses: actions/download-artifact@v3 - with: - name: charmed-mysql-rock - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.GHCR_USER }} - password: ${{ secrets.GHCR_TOKEN }} - - name: Push image to GHCR - run: | - app=$(yq .name rockcraft.yaml) - version=$(yq '(.version)' rockcraft.yaml) - base=$(yq '(.base|split("@"))[1]' rockcraft.yaml) - tag=${version}-${base}_edge - sudo skopeo --insecure-policy copy \ - oci-archive:${app}_${version}_amd64.rock \ - docker-daemon:ghcr.io/canonical/${app}:${tag} - docker push ghcr.io/canonical/${app}:${tag} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..9f6d883 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,62 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. +name: Release to GHCR + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - 8.0-22.04 + workflow_dispatch: + +jobs: + build: + name: Build rock + uses: canonical/data-platform-workflows/.github/workflows/build_rock.yaml@v16 + + release: + name: Release rock + needs: + - build + uses: canonical/data-platform-workflows/.github/workflows/release_rock.yaml@v16 + with: + artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} + permissions: + packages: write # Needed to publish to GitHub Container Registry + + sbom: + name: Generate Software Bill of Materials + needs: + - build + # Run after release so that rock cannot be (maliciously) modified between build & release + - release + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Install Syft + run: curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin + - name: Download rock package(s) + uses: actions/download-artifact@v4 + with: + pattern: ${{ needs.build.outputs.artifact-prefix }}-* + merge-multiple: true + - name: Generate SBOM(s) + shell: python + run: | + import pathlib + import subprocess + + for rock_file in pathlib.Path(".").glob("*.rock"): + subprocess.run( + ["syft", rock_file.name, "--output", f"spdx-json={rock_file.name}.spdx.json"], + check=True, + ) + - name: Upload SBOM(s) + uses: actions/upload-artifact@v4 + with: + name: sbom-${{ needs.build.outputs.artifact-prefix }} + path: '*.spdx.json' + if-no-files-found: error diff --git a/.github/workflows/sbom.yaml b/.github/workflows/sbom.yaml deleted file mode 100644 index 70e796e..0000000 --- a/.github/workflows/sbom.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: Generate SBOM -on: - workflow_call: - -jobs: - build: - uses: ./.github/workflows/build.yaml - sbom: - needs: build - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Install Syft - run: | - curl -sSfL https://raw.githubusercontent.com/anchore/syft/4fc17edd146af34ab06f5b0443ef8ddac3aaf076/install.sh | sh -s -- -b /usr/local/bin - - name: Set version - run: | - version=$(yq '(.version|split("-"))[0]' rockcraft.yaml) - echo "version=${version}" >> "$GITHUB_ENV" - - uses: actions/download-artifact@v3 - with: - name: charmed-mysql-rock - - name: Create SBOM - run: syft charmed-mysql_${{env.version}}_amd64.rock -o spdx-json=charmed-mysql_${{env.version}}_amd64.rock.spdx.json - - name: Upload SBOM - uses: actions/upload-artifact@v3 - with: - path: "charmed-mysql_${{env.version}}_amd64.rock.spdx.json" - name: charmed-mysql_${{env.version}}_amd64.rock.spdx.json diff --git a/.github/workflows/sync_issue_to_jira.yaml b/.github/workflows/sync_issue_to_jira.yaml index 2c1a4f6..b0e0cc2 100644 --- a/.github/workflows/sync_issue_to_jira.yaml +++ b/.github/workflows/sync_issue_to_jira.yaml @@ -9,7 +9,7 @@ on: jobs: sync: name: Sync GitHub issue to Jira - uses: canonical/data-platform-workflows/.github/workflows/sync_issue_to_jira.yaml@v2 + uses: canonical/data-platform-workflows/.github/workflows/sync_issue_to_jira.yaml@v16 with: jira-base-url: https://warthogs.atlassian.net jira-project-key: DPE diff --git a/README.md b/README.md index c9c3a16..5538739 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # MySQL Server ROCK -[![Publish](https://github.com/canonical/charmed-mysql-rock/actions/workflows/publish.yaml/badge.svg)](https://github.com/canonical/charmed-mysql-rock/actions/workflows/publish.yaml) -[![Tests](https://github.com/canonical/charmed-mysql-rock/actions/workflows/integration.yaml/badge.svg)](https://github.com/canonical/charmed-mysql-rock/actions/workflows/integration.yaml) +[![Release to GHCR](https://github.com/canonical/charmed-mysql-rock/actions/workflows/release.yaml/badge.svg)](https://github.com/canonical/charmed-mysql-rock/actions/workflows/release.yaml) This repository contains the packaging metadata for creating a ROCK of MySQL built from the official ubuntu MySQL package from the Ubuntu repository and further installs mysql-shell. For more information on ROCKs, visit the [rockcraft Github](https://github.com/canonical/rockcraft). diff --git a/rockcraft.yaml b/rockcraft.yaml index 23bb21c..9a64d43 100644 --- a/rockcraft.yaml +++ b/rockcraft.yaml @@ -1,15 +1,16 @@ -name: charmed-mysql # the name of your ROCK -base: ubuntu@22.04 # the base environment for this ROCK -version: '8.0.37' # just for humans. Semantic versioning is recommended -summary: Charmed MySQL ROCK OCI # 79 char long summary +name: charmed-mysql +base: ubuntu@22.04 +version: '8.0.37' +summary: Charmed MySQL ROCK OCI description: | MySQL built from the official MySQL package from the MySQL repository and installs mysql-shell. For more information on ROCKs, visit the https://github.com/canonical/rockcraft. -license: Apache-2.0 # your application's SPDX license -platforms: # The platforms this ROCK should be built on and run on +license: Apache-2.0 +platforms: amd64: + arm64: parts: charmed-mysql: @@ -32,30 +33,6 @@ parts: - python3-yaml stage-snaps: - charmed-mysql/8.0/edge - override-stage: | - LIB_DIR=$CRAFT_STAGE/usr/lib/x86_64-linux-gnu - craftctl default - mkdir -p $CRAFT_STAGE/var/lib/mysql - ln -sf $LIB_DIR/libnuma.so.1 \ - /usr/lib/x86_64-linux-gnu/libnuma.so.1 - ln -sf $LIB_DIR/libaio.so.1 \ - /usr/lib/x86_64-linux-gnu/libaio.so.1 - ln -sf $LIB_DIR/libicuuc.so.70 \ - /usr/lib/x86_64-linux-gnu/libicuuc.so.70 - ln -sf $LIB_DIR/libicui18n.so.70 \ - /usr/lib/x86_64-linux-gnu/libicui18n.so.70 - ln -sf $LIB_DIR/libevent_core-2.1.so.7 \ - /usr/lib/x86_64-linux-gnu/libevent_core-2.1.so.7 - ln -sf $LIB_DIR/libevent_pthreads-2.1.so.7 \ - /usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7 - ln -sf $LIB_DIR/libprotobuf-lite.so.23 \ - /usr/lib/x86_64-linux-gnu/libprotobuf-lite.so.23 - ln -sf $LIB_DIR/libicudata.so.70 \ - /usr/lib/x86_64-linux-gnu/libicudata.so.70 - rm -rf /var/lib/mysql/ - mysqld --initialize - cp -r /var/lib/mysql $CRAFT_STAGE/var/lib - chown 584788:584788 $CRAFT_STAGE/var/lib/mysql override-prime: | craftctl default mv -f $CRAFT_PRIME/etc/mysql/mysql.cnf $CRAFT_PRIME/etc/mysql/my.cnf diff --git a/tox.ini b/tox.ini index e3c9218..372ff4f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,7 @@ [tox] requires = tox>=4 -env_list = lint, integration - -[testenv] -setenv = - repo=https://github.com/canonical/mysql-k8s-operator.git - base-name=ghcr.io/canonical/charmed-mysql +env_list = lint [testenv:lint] description = run linters @@ -15,87 +10,3 @@ deps = yamllint commands = yamllint --no-warnings rockcraft.yaml - -[testenv:integration-charm] -description = run operator integration tests -skip_install = true -allowlist_externals = - tox - bash -commands = - bash -ec 'if ! [ -f charmed-mysql_*_amd64.rock ]; then rockcraft pack; fi' {posargs} - bash -ec 'microk8s ctr image import charmed-mysql_*_amd64.rock --base-name {env:base-name}' - bash -ec 'if ! [ -d operator ]; then git clone {env:repo} operator; fi' {posargs} - tox --workdir operator -c operator -e integration-charm - -[testenv:integration-database-relation] -description = run operator integration tests -skip_install = true -allowlist_externals = - tox - bash -commands = - bash -ec 'if ! [ -f charmed-mysql_*_amd64.rock ]; then rockcraft pack; fi' {posargs} - bash -ec 'microk8s ctr image import charmed-mysql_*_amd64.rock --base-name {env:base-name}' - bash -ec 'if ! [ -d operator ]; then git clone {env:repo} operator; fi' {posargs} - tox --workdir operator -c operator -e integration-database-relation - -[testenv:integration-osm-mysql] -description = run operator integration tests -skip_install = true -allowlist_externals = - tox - bash -commands = - bash -ec 'if ! [ -f charmed-mysql_*_amd64.rock ]; then rockcraft pack; fi' {posargs} - bash -ec 'microk8s ctr image import charmed-mysql_*_amd64.rock --base-name {env:base-name}' - bash -ec 'if ! [ -d operator ]; then git clone {env:repo} operator; fi' {posargs} - tox --workdir operator -c operator -e integration-osm-mysql - -[testenv:integration-replication] -description = run operator integration tests -skip_install = true -allowlist_externals = - tox - bash -commands = - bash -ec 'if ! [ -f charmed-mysql_*_amd64.rock ]; then rockcraft pack; fi' {posargs} - bash -ec 'microk8s ctr image import charmed-mysql_*_amd64.rock --base-name {env:base-name}' - bash -ec 'if ! [ -d operator ]; then git clone {env:repo} operator; fi' {posargs} - tox --workdir operator -c operator -e integration-replication - -[testenv:integration-self-healing] -description = run operator integration tests -skip_install = true -allowlist_externals = - tox - bash -commands = - bash -ec 'if ! [ -f charmed-mysql_*_amd64.rock ]; then rockcraft pack; fi' {posargs} - bash -ec 'microk8s ctr image import charmed-mysql_*_amd64.rock --base-name {env:base-name}' - bash -ec 'if ! [ -d operator ]; then git clone {env:repo} operator; fi' {posargs} - tox --workdir operator -c operator -e integration-self-healing - -[testenv:integration-tls] -description = run operator integration tests -skip_install = true -allowlist_externals = - tox - bash -commands = - bash -ec 'if ! [ -f charmed-mysql_*_amd64.rock ]; then rockcraft pack; fi' {posargs} - bash -ec 'microk8s ctr image import charmed-mysql_*_amd64.rock --base-name {env:base-name}' - bash -ec 'if ! [ -d operator ]; then git clone {env:repo} operator; fi' {posargs} - tox --workdir operator -c operator -e integration-tls - -[testenv:integration-backups] -description = run operator integration tests -skip_install = true -allowlist_externals = - tox - bash -commands = - bash -ec 'if ! [ -f charmed-mysql_*_amd64.rock ]; then rockcraft pack; fi' {posargs} - bash -ec 'microk8s ctr image import charmed-mysql_*_amd64.rock --base-name {env:base-name}' - bash -ec 'if ! [ -d operator ]; then git clone {env:repo} operator; fi' {posargs} - tox --workdir operator -c operator -e integration-backups