From 5c7e3f2750310356964e347830ab7bb926b5a6aa Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 7 Jun 2024 09:20:10 -0700 Subject: [PATCH 1/7] Build and Run Arm64 K3s Docker Tests Signed-off-by: Derek Nola --- .github/workflows/build-k3s.yaml | 21 ++++++++++-- .github/workflows/e2e.yaml | 57 +++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-k3s.yaml b/.github/workflows/build-k3s.yaml index 47ba18e247ee..bc69ac7d0337 100644 --- a/.github/workflows/build-k3s.yaml +++ b/.github/workflows/build-k3s.yaml @@ -3,6 +3,10 @@ name: Build K3s on: workflow_call: inputs: + arch: + type: string + description: 'Architecture to build' + default: 'ubuntu-latest' upload-repo: type: boolean required: false @@ -18,11 +22,16 @@ permissions: jobs: build: name: Build - runs-on: ubuntu-latest + runs-on: ${{ inputs.arch }} # defaults to ubuntu-latest, for arm64 use oracle-aarch64-4cpu-16gb timeout-minutes: 20 steps: - name: Checkout K3s uses: actions/checkout@v4 + - name: Setup arm64 Runner + if: contains(inputs.arch, 'aarch64') + run: | + sudo apt-get update + sudo apt-get install -y make - name: Build K3s binary run: | DOCKER_BUILDKIT=1 SKIP_IMAGE=1 SKIP_AIRGAP=1 SKIP_VALIDATE=1 GOCOVER=1 make @@ -44,9 +53,15 @@ jobs: - name: "Save K3s image" if: inputs.upload-image == true run: docker image save rancher/k3s -o ./dist/artifacts/k3s-image.tar - - name: "Upload K3s binary" - if: inputs.upload-repo == false + - name: "Upload K3s Artifacts" + if: inputs.upload-repo == false && inputs.arch == 'ubuntu-latest' uses: actions/upload-artifact@v4 with: name: k3s + path: dist/artifacts/k3s* + - name: "Upload K3s arm64 Artifacts" + if: contains(inputs.arch, 'aarch64') + uses: actions/upload-artifact@v4 + with: + name: k3s-arm64 path: dist/artifacts/k3s* \ No newline at end of file diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 2ba4b3e64e31..eddbc35140d1 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -32,6 +32,12 @@ jobs: uses: ./.github/workflows/build-k3s.yaml with: upload-image: true + build-arm64: + if : github.repository == 'k3s-io/k3s' + uses: ./.github/workflows/build-k3s.yaml + with: + arch: oracle-aarch64-4cpu-16gb + upload-image: true e2e: name: "E2E Tests" needs: build @@ -213,4 +219,53 @@ jobs: ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE -branch=$BRANCH_NAME else ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE - fi \ No newline at end of file + fi + + docker-arm64: + needs: build-arm64 + name: Docker Tests Arm64 + if: github.repository == 'k3s-io/k3s' + runs-on: oracle-aarch64-4cpu-16gb + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + dtest: [basics, bootstraptoken, cacerts, compat, upgrade] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache: false + - name: Install Support Tools + run: | + sudo apt-get update && sudo apt-get install -y wget gawk openssh-client openssh-server net-tools iptables + sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_arm64 -O /usr/bin/yq &&\ + sudo chmod +x /usr/bin/yq + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl" + sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + - name: "Download k3s image" + uses: actions/download-artifact@v4 + with: + name: k3s-arm64 + path: ./dist/artifacts + - name: Load k3s image + run: docker image load -i ./dist/artifacts/k3s-image.tar + - name: Run ${{ matrix.dtest }} Test + run: | + mv ./dist/artifacts/k3s-arm64 ./dist/artifacts/k3s + chmod +x ./dist/artifacts/k3s + . ./scripts/version.sh + . ./tests/docker/test-helpers + . ./tests/docker/test-run-${{ matrix.dtest }} + echo "Did test-run-${{ matrix.dtest }} pass $?" + - name: On Failure, Launch Debug Session + uses: dereknola/action-upterm@v1.1 + if: ${{ failure() }} + with: + ## If no one connects after 5 minutes, shut down server. + wait-timeout-minutes: 5 \ No newline at end of file From 40fe6ee6bd86228c6f29b858129739bb1ea8229b Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 17 Sep 2024 11:14:03 -0700 Subject: [PATCH 2/7] Use native snapshotter on arm64 docker tests Signed-off-by: Derek Nola --- tests/docker/test-run-basics | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/docker/test-run-basics b/tests/docker/test-run-basics index a221b5b7c30e..bcfd6c33a19a 100755 --- a/tests/docker/test-run-basics +++ b/tests/docker/test-run-basics @@ -10,6 +10,8 @@ all_services=( export NUM_SERVERS=1 export NUM_AGENTS=1 export WAIT_SERVICES="${all_services[@]}" +export SERVER_ARGS="--snapshotter=native" +export AGENT_ARGS="--snapshotter=native" start-test() { use-local-storage-volume From e2f3985db5a218c5dc8c5915cfca19c4425f373d Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 17 Sep 2024 13:15:41 -0700 Subject: [PATCH 3/7] Add iptables Signed-off-by: Derek Nola --- tests/docker/test-run-basics | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/docker/test-run-basics b/tests/docker/test-run-basics index bcfd6c33a19a..c3fde55242f8 100755 --- a/tests/docker/test-run-basics +++ b/tests/docker/test-run-basics @@ -10,8 +10,11 @@ all_services=( export NUM_SERVERS=1 export NUM_AGENTS=1 export WAIT_SERVICES="${all_services[@]}" -export SERVER_ARGS="--snapshotter=native" -export AGENT_ARGS="--snapshotter=native" + +if [ "$(uname -i)" = "aarch64" ]; then + export SERVER_ARGS="--snapshotter=native" + export AGENT_ARGS="--snapshotter=native" +fi start-test() { use-local-storage-volume From 4d6e26a8e2fc7a908cf11731be80ac10153038a2 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 6 Dec 2024 11:59:12 -0800 Subject: [PATCH 4/7] Just debug session Signed-off-by: Derek Nola --- .github/workflows/e2e.yaml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index eddbc35140d1..41b9c169c441 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -222,7 +222,7 @@ jobs: fi docker-arm64: - needs: build-arm64 + # needs: build-arm64 name: Docker Tests Arm64 if: github.repository == 'k3s-io/k3s' runs-on: oracle-aarch64-4cpu-16gb @@ -248,24 +248,24 @@ jobs: sudo chmod +x /usr/bin/yq curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl" sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - - name: "Download k3s image" - uses: actions/download-artifact@v4 - with: - name: k3s-arm64 - path: ./dist/artifacts - - name: Load k3s image - run: docker image load -i ./dist/artifacts/k3s-image.tar - - name: Run ${{ matrix.dtest }} Test - run: | - mv ./dist/artifacts/k3s-arm64 ./dist/artifacts/k3s - chmod +x ./dist/artifacts/k3s - . ./scripts/version.sh - . ./tests/docker/test-helpers - . ./tests/docker/test-run-${{ matrix.dtest }} - echo "Did test-run-${{ matrix.dtest }} pass $?" - - name: On Failure, Launch Debug Session + # - name: "Download k3s image" + # uses: actions/download-artifact@v4 + # with: + # name: k3s-arm64 + # path: ./dist/artifacts + # - name: Load k3s image + # run: docker image load -i ./dist/artifacts/k3s-image.tar + # - name: Run ${{ matrix.dtest }} Test + # run: | + # mv ./dist/artifacts/k3s-arm64 ./dist/artifacts/k3s + # chmod +x ./dist/artifacts/k3s + # . ./scripts/version.sh + # . ./tests/docker/test-helpers + # . ./tests/docker/test-run-${{ matrix.dtest }} + # echo "Did test-run-${{ matrix.dtest }} pass $?" + - name: Launch Debug Session uses: dereknola/action-upterm@v1.1 - if: ${{ failure() }} + # if: ${{ failure() }} with: ## If no one connects after 5 minutes, shut down server. wait-timeout-minutes: 5 \ No newline at end of file From d5bf7e9bd12c50e71f22f227a3efa0b1e97b29d0 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 6 Dec 2024 12:04:49 -0800 Subject: [PATCH 5/7] xxx Signed-off-by: Derek Nola --- .github/workflows/e2e.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 41b9c169c441..5049a53ef368 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -264,7 +264,7 @@ jobs: # . ./tests/docker/test-run-${{ matrix.dtest }} # echo "Did test-run-${{ matrix.dtest }} pass $?" - name: Launch Debug Session - uses: dereknola/action-upterm@v1.1 + uses: owenthereal/action-upterm@v1 # if: ${{ failure() }} with: ## If no one connects after 5 minutes, shut down server. From 9c7f5df2a410b4cd9af2aa6850ee8f861da16542 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 6 Dec 2024 12:07:29 -0800 Subject: [PATCH 6/7] xxxx Signed-off-by: Derek Nola --- .github/workflows/e2e.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 5049a53ef368..68707e0bf43a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -248,6 +248,7 @@ jobs: sudo chmod +x /usr/bin/yq curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl" sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + mkdir -p /home/runner/.upterm # - name: "Download k3s image" # uses: actions/download-artifact@v4 # with: @@ -263,6 +264,10 @@ jobs: # . ./tests/docker/test-helpers # . ./tests/docker/test-run-${{ matrix.dtest }} # echo "Did test-run-${{ matrix.dtest }} pass $?" + - name: Check user + run: | + echo "User is $(whoami)" + echo "My Home is $HOME" - name: Launch Debug Session uses: owenthereal/action-upterm@v1 # if: ${{ failure() }} From 5a3cb8b86034d8a1de51182c7c31b79fa09ca1af Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 6 Dec 2024 12:09:50 -0800 Subject: [PATCH 7/7] xv Signed-off-by: Derek Nola --- .github/workflows/e2e.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 68707e0bf43a..baa90ff70303 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -269,8 +269,8 @@ jobs: echo "User is $(whoami)" echo "My Home is $HOME" - name: Launch Debug Session - uses: owenthereal/action-upterm@v1 + uses: mxschmitt/action-tmate@v3 # if: ${{ failure() }} - with: + # with: ## If no one connects after 5 minutes, shut down server. - wait-timeout-minutes: 5 \ No newline at end of file + # wait-timeout-minutes: 5 \ No newline at end of file