From 273e7021159bad7f4689f47524e0aa745c9147d7 Mon Sep 17 00:00:00 2001 From: Jenia Kogan Date: Mon, 30 Sep 2024 12:57:52 +0300 Subject: [PATCH] * Fix the create-linux-wheels * Revert the permissions of the `build_manylinux_wheels_entry_point.sh` back to 755 instead of 644 * Bump the actions/upload-artifacts to v4 * If no files were uploaded - throw an error * Install rust * Add safe dir * Update pip * Store pysh results to a log-file (archive should catch them). * Update actions/python to v5. * Update manylinux variants. * See https://github.com/pypa/manylinux * We should not use EOL manylinux variants. * Add musl_linux_1_2. * Update requeststo 2.23.3, as 2.23.0 was yanked! --- .github/workflows/create-linux-wheels.yml | 26 ++++++++++++++++------- .github/workflows/pr-checker.yml | 16 +++++++------- .github/workflows/static-checks.yml | 2 +- build_manylinux_wheels.sh | 6 ++---- build_manylinux_wheels_entry_point.sh | 13 +++++++++++- changelog.md | 4 +++- dev-README.md | 4 ++-- requirements.txt | 2 +- setup.py | 2 +- tests/dev_tool/test_dev_init.py | 1 + 10 files changed, 49 insertions(+), 27 deletions(-) mode change 100644 => 100755 build_manylinux_wheels_entry_point.sh diff --git a/.github/workflows/create-linux-wheels.yml b/.github/workflows/create-linux-wheels.yml index 4e89189..adc8973 100644 --- a/.github/workflows/create-linux-wheels.yml +++ b/.github/workflows/create-linux-wheels.yml @@ -18,12 +18,17 @@ jobs: if: github.repository == 'PelionIoT/manifest-tool' steps: - uses: actions/checkout@v4 - - run: ./build_manylinux_wheels.sh + + - name: Build wheels + run: ./build_manylinux_wheels.sh + - name: Upload manylinux wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: wheels + name: wheels-manylinux path: dist/*.* + if-no-files-found: error + create-tar-gz: runs-on: ubuntu-22.04 if: github.repository == 'PelionIoT/manifest-tool' @@ -33,10 +38,11 @@ jobs: - run: pip install build - run: python -m build - name: Upload .tar.gz -file - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: wheels + name: wheels-tar-gz path: dist/*.tar.gz + if-no-files-found: error # From internal - do not upload, just build to make sure # all is OK. create-wheels-for-manylinux-internal: @@ -44,13 +50,17 @@ jobs: if: github.repository == 'PelionIoT/manifest-tool-internal' steps: - uses: actions/checkout@v4 - - run: ./build_manylinux_wheels.sh + + - name: Build wheels + run: ./build_manylinux_wheels.sh + - name: Integration with slack uses: act10ns/slack@v2 with: status: ${{ job.status }} - channel: '#testing_device' - if: always() + channel: '#testing_build' + if: failure() + create-tar-gz-int: runs-on: ubuntu-22.04 if: github.repository == 'PelionIoT/manifest-tool-internal' diff --git a/.github/workflows/pr-checker.yml b/.github/workflows/pr-checker.yml index 5b325c2..1b9ce76 100644 --- a/.github/workflows/pr-checker.yml +++ b/.github/workflows/pr-checker.yml @@ -20,16 +20,16 @@ jobs: if: github.repository == 'PelionIoT/manifest-tool' steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.11' - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.9' - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.8' - run: python -m pip install wheel --user @@ -54,16 +54,16 @@ jobs: if: github.repository == 'PelionIoT/manifest-tool-internal' steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.11' - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.9' - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.8' - run: python -m pip install wheel --user diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index 3c29282..4da6f0b 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -51,7 +51,7 @@ jobs: sudo apt-get install black pycodestyle pydocstyle shellcheck python3 black --version rm -rf install-misspell.sh # cause some findings - scripts-internal/pysh-check/pysh-check.sh --workdir . + scripts-internal/pysh-check/pysh-check.sh --workdir . | tee pysh-check.log - name: Archive production artifacts if: always() diff --git a/build_manylinux_wheels.sh b/build_manylinux_wheels.sh index 8b9de33..c28235f 100755 --- a/build_manylinux_wheels.sh +++ b/build_manylinux_wheels.sh @@ -23,10 +23,6 @@ mkdir -p dist rm -rf dist/* # https://github.com/pypa/manylinux -PLAT="manylinux1_x86_64" && docker run --rm -e PLAT="$PLAT" -v "$(pwd)":/io quay.io/pypa/"$PLAT" /io/build_manylinux_wheels_entry_point.sh |& tee "$PLAT.log" & - -PLAT="manylinux2010_x86_64" && docker run --rm -e PLAT="$PLAT" -v "$(pwd)":/io quay.io/pypa/"$PLAT" /io/build_manylinux_wheels_entry_point.sh |& tee "$PLAT.log" & - PLAT="manylinux2014_x86_64" && docker run --rm -e PLAT="$PLAT" -v "$(pwd)":/io quay.io/pypa/"$PLAT" /io/build_manylinux_wheels_entry_point.sh |& tee "$PLAT.log" & # EoL 1st Jan, 2023 @@ -37,6 +33,8 @@ docker run --rm --privileged tonistiigi/binfmt:latest --install all & PLAT="manylinux2014_aarch64" && docker run --rm -e PLAT="$PLAT" -v "$(pwd)":/io quay.io/pypa/"$PLAT" /io/build_manylinux_wheels_entry_point.sh |& tee "$PLAT.log" & +PLAT="musllinux_1_2_x86_64" && docker run --rm -e PLAT="$PLAT" -v "$(pwd)":/io quay.io/pypa/"$PLAT" /io/build_manylinux_wheels_entry_point.sh |& tee "$PLAT.log" & + wait sudo chown -R "$USER":"$USER" dist/ diff --git a/build_manylinux_wheels_entry_point.sh b/build_manylinux_wheels_entry_point.sh old mode 100644 new mode 100755 index 1e369f4..d79088a --- a/build_manylinux_wheels_entry_point.sh +++ b/build_manylinux_wheels_entry_point.sh @@ -1,6 +1,6 @@ #!/bin/bash # ---------------------------------------------------------------------------- -# Copyright 2020-2023 Izuma Networks +# Copyright 2020-2024 Izuma Networks # # SPDX-License-Identifier: Apache-2.0 # @@ -23,6 +23,7 @@ mkdir /work cd /work # Clone from shared dir +git config --global --add safe.directory /io/.git git clone /io manifest-tool cd manifest-tool @@ -78,6 +79,15 @@ for PYBIN in /opt/python/cp3*/bin; do "${PYBIN}/pip" wheel . --no-deps -w wheelhouse/"$PLAT" done +# Turns out we need rust to build the wheels nowaydays +# Check if cargo is available +if command -v cargo; then + echo "Cargo is available => we assume rust is available." +else + echo "Cargo is not available - install rust with rustup" + curl -sSf https://sh.rustup.rs | sh -s -- -y +fi + # Bundle external shared libraries into the wheels for whl in wheelhouse/"$PLAT"/*-linux_x86_64.whl; do repair_wheel "$whl" @@ -100,6 +110,7 @@ for PYBIN in /opt/python/cp3*/bin; do echo '------------------------------------------------------------' echo "${PYBIN}" echo '------------------------------------------------------------' + "${PYBIN}/python" -m pip install --upgrade pip "${PYBIN}/pip" install --prefer-binary -r requirements.txt "${PYBIN}/pip" install manifest-tool --no-index -f wheelhouse/ "${PYBIN}/manifest-tool" --version diff --git a/changelog.md b/changelog.md index 5346a95..9d2def4 100644 --- a/changelog.md +++ b/changelog.md @@ -3,7 +3,9 @@ ## 2.6.2 - Copyright in `update_default_resoruces.c` created with Izuma Networks and current year. - Updated `cryptography` to version `43.0.1`. -- Updated `requests` to version `2.32.0`. +- Updated `requests` to version `2.32.2`. +- Drop `manylinux_1` and `manylinux_2010`, add `musllinux_1_2` as `manylinux` platforms. +- Add updating of pip in wheels building with manylinux. ## 2.6.1 - Drop support for Python 3.7. diff --git a/dev-README.md b/dev-README.md index 2952c38..5d4d951 100644 --- a/dev-README.md +++ b/dev-README.md @@ -118,8 +118,8 @@ the release creation time. 1. Create a PR with the changes to master. Merge it after successful PR checks and approval. 1. Check that the `PR-checker` workflow passes on the master branch. It runs all the tox tests on all the OSes. 1. Check that the `create-Linux-wheels` workflow passes on the master branch. It builds the Linux wheels and uploads them as an artifact. - 1. Download the `wheels.zip` artifact to your local machine from the `create-Linux-wheels` workflow. - 1. Extract the `wheels.zip` to a folder - + 1. Download the `wheels-manylinux.zip` and `wheels-tar-gz.zip `artifacts to your local machine from the `create-Linux-wheels` workflow. + 1. Extract both zip files to a folder. 1. Install `twine`: `pip install twine`. 1. Publish to https://test.pypi.org and check: ``` diff --git a/requirements.txt b/requirements.txt index 1617270..e087400 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ asn1ate>=0.5,<=0.6.0 cryptography==43.0.1 jsonschema>=2.6.0,<=3.2.0 pyasn1>=0.3.1,<=0.4.8 -requests==2.32 \ No newline at end of file +requests==2.32.3 \ No newline at end of file diff --git a/setup.py b/setup.py index b4d262a..7611505 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ # ---------------------------------------------------------------------------- # Copyright 2019-2021 Pelion -# Copyright (c) 2022-2023 Izuma Networks +# Copyright (c) 2022-2024 Izuma Networks # # SPDX-License-Identifier: Apache-2.0 # diff --git a/tests/dev_tool/test_dev_init.py b/tests/dev_tool/test_dev_init.py index b2d8aa5..b491614 100644 --- a/tests/dev_tool/test_dev_init.py +++ b/tests/dev_tool/test_dev_init.py @@ -1,5 +1,6 @@ # ---------------------------------------------------------------------------- # Copyright 2019-2021 Pelion +# Copyright 2022-2024 Izuma Networks # # SPDX-License-Identifier: Apache-2.0 #