From 500db436a32aecd0ce220e8dc0bc09317fd180a7 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Thu, 12 Sep 2024 22:04:27 +0000 Subject: [PATCH 01/38] adding graviton docker image --- .github/workflows/docker-nightly-build.yml | 32 ++++++- docker/build_image.sh | 8 +- docker/docker_nightly.py | 102 +++++++++++++-------- 3 files changed, 99 insertions(+), 43 deletions(-) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index a46bf30653..6b6c0a12e2 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -1,10 +1,11 @@ name: Push Docker Nightly on: - # run every day at 1:15pm + # Run every day at 1:15pm schedule: - cron: "15 13 * * *" workflow_dispatch: + jobs: nightly: runs-on: [self-hosted, ci-gpu] @@ -37,7 +38,6 @@ jobs: run: | cd kubernetes/kserve python docker_nightly.py --cleanup - - name: Open issue on failure if: ${{ failure() && github.event_name == 'schedule' }} uses: dacbd/create-issue-action@v1 @@ -46,3 +46,31 @@ jobs: title: Nightly Docker build failed body: Commit ${{ github.sha }} daily scheduled [CI run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) failed, please check why assignees: "" + + graviton_job: + runs-on: [graviton-test] + steps: + - name: Clean up previous run on Graviton + run: | + echo "Cleaning up previous Graviton" + ls -la ./ + sudo rm -rf ./* || true + sudo rm -rf ./.??* || true + ls -la ./ + - name: Setup Python 3.9 on Graviton + uses: actions/setup-python@v5 + with: + python-version: 3.9 + architecture: aarch64 + - name: Checkout TorchServe + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Login to Docker + env: + DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} + run: docker login --username pytorchbot --password "$DOCKER_PASSWORD" + - name: Push Docker Nightly + run: | + cd docker + python docker_nightly.py --cleanup --arm diff --git a/docker/build_image.sh b/docker/build_image.sh index 650215a492..89801a63e5 100755 --- a/docker/build_image.sh +++ b/docker/build_image.sh @@ -18,6 +18,7 @@ BUILD_NIGHTLY=false BUILD_FROM_SRC=false LOCAL_CHANGES=true PYTHON_VERSION=3.9 +ARCH="linux/amd64" for arg in "$@" do @@ -39,6 +40,7 @@ do echo "-n, --nightly specify to build with TorchServe nightly" echo "-s, --source specify to build with TorchServe from source" echo "-r, --remote specify to use local TorchServe" + echo "-a, --arch to arm64" exit 0 ;; -b|--branch_name) @@ -101,6 +103,10 @@ do BUILD_CPP=true shift ;; + -a|--arch) + ARCH="linux/arm64" + shift + ;; -n|--nightly) BUILD_NIGHTLY=true shift @@ -218,7 +224,7 @@ if [ "${BUILD_TYPE}" == "production" ] then DOCKER_BUILDKIT=1 docker build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ - --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --target production-image ../ + --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform "${ARCH}" --target production-image ../ elif [ "${BUILD_TYPE}" == "ci" ] then DOCKER_BUILDKIT=1 docker build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ diff --git a/docker/docker_nightly.py b/docker/docker_nightly.py index 56479f22b2..e68f0d9733 100644 --- a/docker/docker_nightly.py +++ b/docker/docker_nightly.py @@ -28,60 +28,82 @@ action="store_true", help="Delete all built docker images", ) + parser.add_argument("--arm", action="store_true", help="Sets architecture to arm") args = parser.parse_args() dry_run = args.dry_run organization = args.organization project = "torchserve-nightly" cpu_version = f"{project}:cpu-{get_nightly_version()}" + graviton_version = f"{project}:cpu-graviton-{get_nightly_version()}" gpu_version = f"{project}:gpu-{get_nightly_version()}" cpp_dev_cpu_version = f"{project}:cpp-dev-cpu-{get_nightly_version()}" cpp_dev_gpu_version = f"{project}:cpp-dev-gpu-{get_nightly_version()}" # Build Nightly images and append the date in the name - try_and_handle(f"./build_image.sh -n -t {organization}/{cpu_version}", dry_run) - try_and_handle( - f"./build_image.sh -g -cv cu121 -n -t {organization}/{gpu_version}", - dry_run, - ) - try_and_handle( - f"./build_image.sh -bt dev -cpp -t {organization}/{cpp_dev_cpu_version}", - dry_run, - ) - try_and_handle( - f"./build_image.sh -bt dev -g -cv cu121 -cpp -t {organization}/{cpp_dev_gpu_version}", - dry_run, - ) + if args.arch: + try_and_handle( + f"./build_image.sh -n -a -t {organization}/{graviton_version}", dry_run + ) - # Push Nightly images to official PyTorch Dockerhub account - try_and_handle(f"docker push {organization}/{cpu_version}", dry_run) - try_and_handle(f"docker push {organization}/{gpu_version}", dry_run) - try_and_handle(f"docker push {organization}/{cpp_dev_cpu_version}", dry_run) - try_and_handle(f"docker push {organization}/{cpp_dev_gpu_version}", dry_run) + try_and_handle(f"docker push {organization}/{graviton_version}", dry_run) - # Tag nightly images with latest - try_and_handle( - f"docker tag {organization}/{cpu_version} {organization}/{project}:latest-cpu", - dry_run, - ) - try_and_handle( - f"docker tag {organization}/{gpu_version} {organization}/{project}:latest-gpu", - dry_run, - ) - try_and_handle( - f"docker tag {organization}/{cpp_dev_cpu_version} {organization}/{project}:latest-cpp-dev-cpu", - dry_run, - ) - try_and_handle( - f"docker tag {organization}/{cpp_dev_gpu_version} {organization}/{project}:latest-cpp-dev-gpu", - dry_run, - ) + try_and_handle( + f"docker tag {organization}/{cpu_version} {graviton_version}/{project}:latest-cpu-graviton", + dry_run, + ) + + try_and_handle( + f"docker push {organization}/{project}:latest-cpu-graviton", dry_run + ) + else: + try_and_handle(f"./build_image.sh -n -t {organization}/{cpu_version}", dry_run) + try_and_handle( + f"./build_image.sh -g -cv cu121 -n -t {organization}/{gpu_version}", + dry_run, + ) + try_and_handle( + f"./build_image.sh -bt dev -cpp -t {organization}/{cpp_dev_cpu_version}", + dry_run, + ) + try_and_handle( + f"./build_image.sh -bt dev -g -cv cu121 -cpp -t {organization}/{cpp_dev_gpu_version}", + dry_run, + ) + + # Push Nightly images to official PyTorch Dockerhub account + try_and_handle(f"docker push {organization}/{cpu_version}", dry_run) + try_and_handle(f"docker push {organization}/{gpu_version}", dry_run) + try_and_handle(f"docker push {organization}/{cpp_dev_cpu_version}", dry_run) + try_and_handle(f"docker push {organization}/{cpp_dev_gpu_version}", dry_run) + + # Tag nightly images with latest + try_and_handle( + f"docker tag {organization}/{cpu_version} {organization}/{project}:latest-cpu", + dry_run, + ) + try_and_handle( + f"docker tag {organization}/{gpu_version} {organization}/{project}:latest-gpu", + dry_run, + ) + try_and_handle( + f"docker tag {organization}/{cpp_dev_cpu_version} {organization}/{project}:latest-cpp-dev-cpu", + dry_run, + ) + try_and_handle( + f"docker tag {organization}/{cpp_dev_gpu_version} {organization}/{project}:latest-cpp-dev-gpu", + dry_run, + ) - # Push images with latest tag - try_and_handle(f"docker push {organization}/{project}:latest-cpu", dry_run) - try_and_handle(f"docker push {organization}/{project}:latest-gpu", dry_run) - try_and_handle(f"docker push {organization}/{project}:latest-cpp-dev-cpu", dry_run) - try_and_handle(f"docker push {organization}/{project}:latest-cpp-dev-gpu", dry_run) + # Push images with latest tag + try_and_handle(f"docker push {organization}/{project}:latest-cpu", dry_run) + try_and_handle(f"docker push {organization}/{project}:latest-gpu", dry_run) + try_and_handle( + f"docker push {organization}/{project}:latest-cpp-dev-cpu", dry_run + ) + try_and_handle( + f"docker push {organization}/{project}:latest-cpp-dev-gpu", dry_run + ) # Cleanup built images if args.cleanup: From 42491358270456087a8d694b56a5e29234f1ecc6 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Fri, 13 Sep 2024 22:54:28 +0000 Subject: [PATCH 02/38] testing multiplatform ci --- .github/workflows/docker-nightly-build.yml | 64 ++++--------- docker/build_image.sh | 21 +++-- docker/docker_nightly.py | 102 +++++++++------------ 3 files changed, 74 insertions(+), 113 deletions(-) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index 6b6c0a12e2..e5a0eb5852 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -2,9 +2,11 @@ name: Push Docker Nightly on: # Run every day at 1:15pm - schedule: - - cron: "15 13 * * *" - workflow_dispatch: + # schedule: + # - cron: "15 13 * * *" + # workflow_dispatch: + push: + branches: "docker_aarch" jobs: nightly: @@ -28,49 +30,21 @@ jobs: submodules: recursive - name: Login to Docker env: - DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} - run: docker login --username pytorchbot --password "$DOCKER_PASSWORD" + DOCKER_PASSWORD: ${{secrets.UDAI_TEST}} + run: docker login --username udaij12 --password "$DOCKER_PASSWORD" - name: Push Docker Nightly run: | cd docker python docker_nightly.py --cleanup - - name: Push KServe Docker Nightly - run: | - cd kubernetes/kserve - python docker_nightly.py --cleanup - - name: Open issue on failure - if: ${{ failure() && github.event_name == 'schedule' }} - uses: dacbd/create-issue-action@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - title: Nightly Docker build failed - body: Commit ${{ github.sha }} daily scheduled [CI run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) failed, please check why - assignees: "" - - graviton_job: - runs-on: [graviton-test] - steps: - - name: Clean up previous run on Graviton - run: | - echo "Cleaning up previous Graviton" - ls -la ./ - sudo rm -rf ./* || true - sudo rm -rf ./.??* || true - ls -la ./ - - name: Setup Python 3.9 on Graviton - uses: actions/setup-python@v5 - with: - python-version: 3.9 - architecture: aarch64 - - name: Checkout TorchServe - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Login to Docker - env: - DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} - run: docker login --username pytorchbot --password "$DOCKER_PASSWORD" - - name: Push Docker Nightly - run: | - cd docker - python docker_nightly.py --cleanup --arm + # - name: Push KServe Docker Nightly + # run: | + # cd kubernetes/kserve + # python docker_nightly.py --cleanup + # - name: Open issue on failure + # if: ${{ failure() && github.event_name == 'schedule' }} + # uses: dacbd/create-issue-action@v1 + # with: + # token: ${{ secrets.GITHUB_TOKEN }} + # title: Nightly Docker build failed + # body: Commit ${{ github.sha }} daily scheduled [CI run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) failed, please check why + # assignees: "" diff --git a/docker/build_image.sh b/docker/build_image.sh index 89801a63e5..af371f495c 100755 --- a/docker/build_image.sh +++ b/docker/build_image.sh @@ -18,7 +18,8 @@ BUILD_NIGHTLY=false BUILD_FROM_SRC=false LOCAL_CHANGES=true PYTHON_VERSION=3.9 -ARCH="linux/amd64" +ARCH="linux/amd64,linux/arm64" +MULTI=false for arg in "$@" do @@ -40,7 +41,6 @@ do echo "-n, --nightly specify to build with TorchServe nightly" echo "-s, --source specify to build with TorchServe from source" echo "-r, --remote specify to use local TorchServe" - echo "-a, --arch to arm64" exit 0 ;; -b|--branch_name) @@ -103,8 +103,8 @@ do BUILD_CPP=true shift ;; - -a|--arch) - ARCH="linux/arm64" + -m|--multi) + MULTI=true shift ;; -n|--nightly) @@ -221,10 +221,15 @@ then fi if [ "${BUILD_TYPE}" == "production" ] -then - DOCKER_BUILDKIT=1 docker build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ - --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ - --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform "${ARCH}" --target production-image ../ + if [ "${MULTI}" == "true"] + then + DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ + --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ + --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform "${ARCH}" --target production-image ../ --push + else + DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ + --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ + --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --target production-image ../ elif [ "${BUILD_TYPE}" == "ci" ] then DOCKER_BUILDKIT=1 docker build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ diff --git a/docker/docker_nightly.py b/docker/docker_nightly.py index e68f0d9733..bf99ef128a 100644 --- a/docker/docker_nightly.py +++ b/docker/docker_nightly.py @@ -28,82 +28,64 @@ action="store_true", help="Delete all built docker images", ) - parser.add_argument("--arm", action="store_true", help="Sets architecture to arm") args = parser.parse_args() dry_run = args.dry_run organization = args.organization project = "torchserve-nightly" cpu_version = f"{project}:cpu-{get_nightly_version()}" - graviton_version = f"{project}:cpu-graviton-{get_nightly_version()}" gpu_version = f"{project}:gpu-{get_nightly_version()}" cpp_dev_cpu_version = f"{project}:cpp-dev-cpu-{get_nightly_version()}" cpp_dev_gpu_version = f"{project}:cpp-dev-gpu-{get_nightly_version()}" # Build Nightly images and append the date in the name - if args.arch: - try_and_handle( - f"./build_image.sh -n -a -t {organization}/{graviton_version}", dry_run - ) - try_and_handle(f"docker push {organization}/{graviton_version}", dry_run) - - try_and_handle( - f"docker tag {organization}/{cpu_version} {graviton_version}/{project}:latest-cpu-graviton", - dry_run, - ) + try_and_handle(f"./build_image.sh -m -n -t {organization}/{cpu_version}", dry_run) + try_and_handle( + f"./build_image.sh -g -cv cu121 -n -t {organization}/{gpu_version}", + dry_run, + ) + # try_and_handle( + # f"./build_image.sh -bt dev -cpp -t {organization}/{cpp_dev_cpu_version}", + # dry_run, + # ) + # try_and_handle( + # f"./build_image.sh -bt dev -g -cv cu121 -cpp -t {organization}/{cpp_dev_gpu_version}", + # dry_run, + # ) - try_and_handle( - f"docker push {organization}/{project}:latest-cpu-graviton", dry_run - ) - else: - try_and_handle(f"./build_image.sh -n -t {organization}/{cpu_version}", dry_run) - try_and_handle( - f"./build_image.sh -g -cv cu121 -n -t {organization}/{gpu_version}", - dry_run, - ) - try_and_handle( - f"./build_image.sh -bt dev -cpp -t {organization}/{cpp_dev_cpu_version}", - dry_run, - ) - try_and_handle( - f"./build_image.sh -bt dev -g -cv cu121 -cpp -t {organization}/{cpp_dev_gpu_version}", - dry_run, - ) + # Push Nightly images to official PyTorch Dockerhub account + try_and_handle(f"docker push {organization}/{gpu_version}", dry_run) + # try_and_handle(f"docker push {organization}/{cpp_dev_cpu_version}", dry_run) + # try_and_handle(f"docker push {organization}/{cpp_dev_gpu_version}", dry_run) - # Push Nightly images to official PyTorch Dockerhub account - try_and_handle(f"docker push {organization}/{cpu_version}", dry_run) - try_and_handle(f"docker push {organization}/{gpu_version}", dry_run) - try_and_handle(f"docker push {organization}/{cpp_dev_cpu_version}", dry_run) - try_and_handle(f"docker push {organization}/{cpp_dev_gpu_version}", dry_run) + # Tag nightly images with latest + try_and_handle( + f"docker buildx imagetools create --tag {organization}/{project}:latest-cpu {organization}/{cpu_version}", + dry_run, + ) + try_and_handle( + f"docker tag {organization}/{gpu_version} {organization}/{project}:latest-gpu", + dry_run, + ) + # try_and_handle( + # f"docker tag {organization}/{cpp_dev_cpu_version} {organization}/{project}:latest-cpp-dev-cpu", + # dry_run, + # ) + # try_and_handle( + # f"docker tag {organization}/{cpp_dev_gpu_version} {organization}/{project}:latest-cpp-dev-gpu", + # dry_run, + # ) - # Tag nightly images with latest - try_and_handle( - f"docker tag {organization}/{cpu_version} {organization}/{project}:latest-cpu", - dry_run, - ) - try_and_handle( - f"docker tag {organization}/{gpu_version} {organization}/{project}:latest-gpu", - dry_run, - ) - try_and_handle( - f"docker tag {organization}/{cpp_dev_cpu_version} {organization}/{project}:latest-cpp-dev-cpu", - dry_run, - ) - try_and_handle( - f"docker tag {organization}/{cpp_dev_gpu_version} {organization}/{project}:latest-cpp-dev-gpu", - dry_run, - ) + # Push images with latest tag + try_and_handle(f"docker push {organization}/{project}:latest-gpu", dry_run) - # Push images with latest tag - try_and_handle(f"docker push {organization}/{project}:latest-cpu", dry_run) - try_and_handle(f"docker push {organization}/{project}:latest-gpu", dry_run) - try_and_handle( - f"docker push {organization}/{project}:latest-cpp-dev-cpu", dry_run - ) - try_and_handle( - f"docker push {organization}/{project}:latest-cpp-dev-gpu", dry_run - ) + # try_and_handle( + # f"docker push {organization}/{project}:latest-cpp-dev-cpu", dry_run + # ) + # try_and_handle( + # f"docker push {organization}/{project}:latest-cpp-dev-gpu", dry_run + # ) # Cleanup built images if args.cleanup: From f62f1586850cb4dabf091809ed559afad07df565 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Fri, 13 Sep 2024 22:57:10 +0000 Subject: [PATCH 03/38] testing multiplatform ci --- docker/build_image.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docker/build_image.sh b/docker/build_image.sh index af371f495c..d7d7de280b 100755 --- a/docker/build_image.sh +++ b/docker/build_image.sh @@ -220,17 +220,17 @@ then fi fi -if [ "${BUILD_TYPE}" == "production" ] - if [ "${MULTI}" == "true"] - then - DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ +if [ "${BUILD_TYPE}" == "production" ]; then + if [ "${MULTI}" == "true" ]; then + DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ - --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform "${ARCH}" --target production-image ../ --push + --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform "${ARCH}" --target production-image ../ --push else - DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ + DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ - --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --target production-image ../ -elif [ "${BUILD_TYPE}" == "ci" ] + --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --target production-image ../ + fi +elif [ "${BUILD_TYPE}" == "ci" ]; then then DOCKER_BUILDKIT=1 docker build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ From b2775eb3c7ea1244b702b1fe16fed9a10a612303 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Fri, 13 Sep 2024 22:59:10 +0000 Subject: [PATCH 04/38] testing multiplatform ci --- docker/build_image.sh | 2 +- docker/docker_nightly.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build_image.sh b/docker/build_image.sh index d7d7de280b..6a6089e0e3 100755 --- a/docker/build_image.sh +++ b/docker/build_image.sh @@ -230,7 +230,7 @@ if [ "${BUILD_TYPE}" == "production" ]; then --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --target production-image ../ fi -elif [ "${BUILD_TYPE}" == "ci" ]; then +elif [ "${BUILD_TYPE}" == "ci" ]; then DOCKER_BUILDKIT=1 docker build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ diff --git a/docker/docker_nightly.py b/docker/docker_nightly.py index bf99ef128a..271c8a6eb5 100644 --- a/docker/docker_nightly.py +++ b/docker/docker_nightly.py @@ -15,7 +15,7 @@ parser.add_argument( "--organization", type=str, - default="pytorch", + default="udaij12", help="The name of the Dockerhub organization where the images will be pushed", ) parser.add_argument( From 45df7945bfb864b4a52689b3510ca428f6fc82f8 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Fri, 13 Sep 2024 23:00:49 +0000 Subject: [PATCH 05/38] adding new builder --- .github/workflows/docker-nightly-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index e5a0eb5852..9e7f7c9c41 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -35,6 +35,7 @@ jobs: - name: Push Docker Nightly run: | cd docker + docker buildx create --name multibuilder --use python docker_nightly.py --cleanup # - name: Push KServe Docker Nightly # run: | From 761955222b5b9d397649ede01e57bbe94a2b1e34 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Fri, 13 Sep 2024 23:11:57 +0000 Subject: [PATCH 06/38] removing arm --- .github/workflows/docker-nightly-build.yml | 2 +- docker/build_image.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index 9e7f7c9c41..659b1beeff 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -35,7 +35,7 @@ jobs: - name: Push Docker Nightly run: | cd docker - docker buildx create --name multibuilder --use + docker buildx use multibuilder python docker_nightly.py --cleanup # - name: Push KServe Docker Nightly # run: | diff --git a/docker/build_image.sh b/docker/build_image.sh index 6a6089e0e3..57732a861f 100755 --- a/docker/build_image.sh +++ b/docker/build_image.sh @@ -18,7 +18,7 @@ BUILD_NIGHTLY=false BUILD_FROM_SRC=false LOCAL_CHANGES=true PYTHON_VERSION=3.9 -ARCH="linux/amd64,linux/arm64" +ARCH="linux/amd64" MULTI=false for arg in "$@" From 33cabbcf2e59b3af01cd0b38dc6bed5682274e7a Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Fri, 13 Sep 2024 23:12:38 +0000 Subject: [PATCH 07/38] removing arm --- .github/workflows/docker-nightly-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index 659b1beeff..9e7f7c9c41 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -35,7 +35,7 @@ jobs: - name: Push Docker Nightly run: | cd docker - docker buildx use multibuilder + docker buildx create --name multibuilder --use python docker_nightly.py --cleanup # - name: Push KServe Docker Nightly # run: | From debdb7f29582f8d2e113b204d96a844a4b1786a9 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Fri, 13 Sep 2024 23:23:53 +0000 Subject: [PATCH 08/38] testing arm --- docker/build_image.sh | 2 +- docker/docker_nightly.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build_image.sh b/docker/build_image.sh index 57732a861f..6a6089e0e3 100755 --- a/docker/build_image.sh +++ b/docker/build_image.sh @@ -18,7 +18,7 @@ BUILD_NIGHTLY=false BUILD_FROM_SRC=false LOCAL_CHANGES=true PYTHON_VERSION=3.9 -ARCH="linux/amd64" +ARCH="linux/amd64,linux/arm64" MULTI=false for arg in "$@" diff --git a/docker/docker_nightly.py b/docker/docker_nightly.py index 271c8a6eb5..73b9bc2562 100644 --- a/docker/docker_nightly.py +++ b/docker/docker_nightly.py @@ -65,7 +65,7 @@ dry_run, ) try_and_handle( - f"docker tag {organization}/{gpu_version} {organization}/{project}:latest-gpu", + f"docker buildx imagetools create --tag {organization}/{project}:latest-gpu {organization}/{gpu_version}", dry_run, ) # try_and_handle( From 1a6d0aa0c73c0c5b0da1d775188bd008c846b1bc Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Fri, 13 Sep 2024 23:29:00 +0000 Subject: [PATCH 09/38] tests --- .github/workflows/docker-nightly-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index 9e7f7c9c41..7ac237f9ba 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -35,6 +35,7 @@ jobs: - name: Push Docker Nightly run: | cd docker + docker buildx rm multibuilder docker buildx create --name multibuilder --use python docker_nightly.py --cleanup # - name: Push KServe Docker Nightly From b8637a36c94a42863ada144619559ac8449a4dd0 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Fri, 13 Sep 2024 23:48:13 +0000 Subject: [PATCH 10/38] testing driver command --- .github/workflows/docker-nightly-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index 7ac237f9ba..d1ad24ffe4 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -36,7 +36,7 @@ jobs: run: | cd docker docker buildx rm multibuilder - docker buildx create --name multibuilder --use + docker buildx create --name multibuilder --use --driver docker-container --bootstrap python docker_nightly.py --cleanup # - name: Push KServe Docker Nightly # run: | From 3244d4844b5c4e498aa6276ebaea9aa282eebec9 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Fri, 13 Sep 2024 23:56:21 +0000 Subject: [PATCH 11/38] testing driver command --- .github/workflows/docker-nightly-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index d1ad24ffe4..add341b3fa 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -35,6 +35,7 @@ jobs: - name: Push Docker Nightly run: | cd docker + sudo apt-get update docker buildx rm multibuilder docker buildx create --name multibuilder --use --driver docker-container --bootstrap python docker_nightly.py --cleanup From 528aff08ed70ffdaec4038cfb9d51f7bd86d603f Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sat, 14 Sep 2024 21:32:31 +0000 Subject: [PATCH 12/38] testing on newer instance --- .github/workflows/docker-nightly-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index add341b3fa..3847c8e242 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -10,7 +10,7 @@ on: jobs: nightly: - runs-on: [self-hosted, ci-gpu] + runs-on: [self-hosted, multi-test] steps: - name: Clean up previous run run: | From d43e74e9f0f580cd19fdd9abfa8068346019fee5 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sat, 14 Sep 2024 21:34:23 +0000 Subject: [PATCH 13/38] testing on newer instance --- .github/workflows/docker-nightly-build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index 3847c8e242..80f93f7662 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -36,7 +36,6 @@ jobs: run: | cd docker sudo apt-get update - docker buildx rm multibuilder docker buildx create --name multibuilder --use --driver docker-container --bootstrap python docker_nightly.py --cleanup # - name: Push KServe Docker Nightly From e54ee1886904d23ab76b6a981ef23b2a5b03f9a9 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sat, 14 Sep 2024 21:41:22 +0000 Subject: [PATCH 14/38] testing newer --- .github/workflows/docker-nightly-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index 80f93f7662..cc1a4b787f 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -36,6 +36,8 @@ jobs: run: | cd docker sudo apt-get update + sudo apt-get install --reinstall python3.9 python3-distutils python3.9-dev python3.9-venv -y + docker buildx rm multibuilder docker buildx create --name multibuilder --use --driver docker-container --bootstrap python docker_nightly.py --cleanup # - name: Push KServe Docker Nightly From b79766dcfbd18e47bfc22fd1867c15bfdfb0c64f Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sat, 14 Sep 2024 21:51:05 +0000 Subject: [PATCH 15/38] rm command --- .github/workflows/docker-nightly-build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index cc1a4b787f..3847c8e242 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -36,7 +36,6 @@ jobs: run: | cd docker sudo apt-get update - sudo apt-get install --reinstall python3.9 python3-distutils python3.9-dev python3.9-venv -y docker buildx rm multibuilder docker buildx create --name multibuilder --use --driver docker-container --bootstrap python docker_nightly.py --cleanup From 76e2a4c48029949cc4101ae47305b18eb8001d7f Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sun, 15 Sep 2024 03:10:05 +0000 Subject: [PATCH 16/38] changing platform --- docker/build_image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build_image.sh b/docker/build_image.sh index 6a6089e0e3..a78a09025e 100755 --- a/docker/build_image.sh +++ b/docker/build_image.sh @@ -18,7 +18,7 @@ BUILD_NIGHTLY=false BUILD_FROM_SRC=false LOCAL_CHANGES=true PYTHON_VERSION=3.9 -ARCH="linux/amd64,linux/arm64" +ARCH="linux/arm64" MULTI=false for arg in "$@" @@ -224,7 +224,7 @@ if [ "${BUILD_TYPE}" == "production" ]; then if [ "${MULTI}" == "true" ]; then DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ - --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform "${ARCH}" --target production-image ../ --push + --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform linux/amd64,linux/arm64 --target production-image ../ --push else DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ From 61d523e959e1e0b5703d215046c879ccd957ea4f Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sun, 15 Sep 2024 03:38:55 +0000 Subject: [PATCH 17/38] testing only amd --- docker/build_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build_image.sh b/docker/build_image.sh index a78a09025e..8a45e93bc6 100755 --- a/docker/build_image.sh +++ b/docker/build_image.sh @@ -224,7 +224,7 @@ if [ "${BUILD_TYPE}" == "production" ]; then if [ "${MULTI}" == "true" ]; then DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ - --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform linux/amd64,linux/arm64 --target production-image ../ --push + --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform linux/amd64 --target production-image ../ --push else DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ From ea4b4f904476761c2d52f3c7283b08f30271f323 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sun, 15 Sep 2024 03:53:50 +0000 Subject: [PATCH 18/38] testing both arch --- docker/build_image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build_image.sh b/docker/build_image.sh index 8a45e93bc6..1111f05816 100755 --- a/docker/build_image.sh +++ b/docker/build_image.sh @@ -224,11 +224,11 @@ if [ "${BUILD_TYPE}" == "production" ]; then if [ "${MULTI}" == "true" ]; then DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ - --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform linux/amd64 --target production-image ../ --push + --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform linux/amd64,linux/arm64 --target production-image ../ --push else DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ - --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --target production-image ../ + --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --target production-image ../ --load fi elif [ "${BUILD_TYPE}" == "ci" ]; then From 494cff96972fbda3831f16617a72c96fb7676c21 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sun, 15 Sep 2024 03:56:55 +0000 Subject: [PATCH 19/38] testing both arch --- docker/build_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build_image.sh b/docker/build_image.sh index 1111f05816..e132480315 100755 --- a/docker/build_image.sh +++ b/docker/build_image.sh @@ -224,7 +224,7 @@ if [ "${BUILD_TYPE}" == "production" ]; then if [ "${MULTI}" == "true" ]; then DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ - --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform linux/amd64,linux/arm64 --target production-image ../ --push + --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform linux/arm64 --target production-image ../ --push else DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ From 97d5c4cd7c998c1a220620ef40276ff548739f34 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sun, 15 Sep 2024 04:29:57 +0000 Subject: [PATCH 20/38] testing both --- docker/build_image.sh | 2 +- docker/docker_nightly.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build_image.sh b/docker/build_image.sh index e132480315..1111f05816 100755 --- a/docker/build_image.sh +++ b/docker/build_image.sh @@ -224,7 +224,7 @@ if [ "${BUILD_TYPE}" == "production" ]; then if [ "${MULTI}" == "true" ]; then DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ - --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform linux/arm64 --target production-image ../ --push + --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform linux/amd64,linux/arm64 --target production-image ../ --push else DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ diff --git a/docker/docker_nightly.py b/docker/docker_nightly.py index 73b9bc2562..a06b1184ea 100644 --- a/docker/docker_nightly.py +++ b/docker/docker_nightly.py @@ -78,7 +78,7 @@ # ) # Push images with latest tag - try_and_handle(f"docker push {organization}/{project}:latest-gpu", dry_run) + # try_and_handle(f"docker push {organization}/{project}:latest-gpu", dry_run) # try_and_handle( # f"docker push {organization}/{project}:latest-cpp-dev-cpu", dry_run From 3a985ab765d01139db2d40e6123c7ead00f93b97 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sun, 15 Sep 2024 04:51:56 +0000 Subject: [PATCH 21/38] remove builder --- .github/workflows/docker-nightly-build.yml | 1 - docker/build_image.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index 3847c8e242..80f93f7662 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -36,7 +36,6 @@ jobs: run: | cd docker sudo apt-get update - docker buildx rm multibuilder docker buildx create --name multibuilder --use --driver docker-container --bootstrap python docker_nightly.py --cleanup # - name: Push KServe Docker Nightly diff --git a/docker/build_image.sh b/docker/build_image.sh index 1111f05816..e132480315 100755 --- a/docker/build_image.sh +++ b/docker/build_image.sh @@ -224,7 +224,7 @@ if [ "${BUILD_TYPE}" == "production" ]; then if [ "${MULTI}" == "true" ]; then DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ - --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform linux/amd64,linux/arm64 --target production-image ../ --push + --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform linux/arm64 --target production-image ../ --push else DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ From bee430b451da40ea2327289a828e3351ebe49513 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sun, 15 Sep 2024 04:53:20 +0000 Subject: [PATCH 22/38] remove builder --- .github/workflows/docker-nightly-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index 80f93f7662..cec5fdf875 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -36,7 +36,7 @@ jobs: run: | cd docker sudo apt-get update - docker buildx create --name multibuilder --use --driver docker-container --bootstrap + docker buildx use multibuilder python docker_nightly.py --cleanup # - name: Push KServe Docker Nightly # run: | From 2e619c275c090b6409d0977d8fdbcb7f1333efda Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sun, 15 Sep 2024 04:57:06 +0000 Subject: [PATCH 23/38] adding amd --- docker/build_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build_image.sh b/docker/build_image.sh index e132480315..1111f05816 100755 --- a/docker/build_image.sh +++ b/docker/build_image.sh @@ -224,7 +224,7 @@ if [ "${BUILD_TYPE}" == "production" ]; then if [ "${MULTI}" == "true" ]; then DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ - --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform linux/arm64 --target production-image ../ --push + --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform linux/amd64,linux/arm64 --target production-image ../ --push else DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ From a23b77d359e4ab53634b8acade6423469d688157 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sun, 15 Sep 2024 06:15:09 +0000 Subject: [PATCH 24/38] building cache --- .github/workflows/docker-nightly-build.yml | 2 +- docker/build_image.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index cec5fdf875..73ed1933eb 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -10,7 +10,7 @@ on: jobs: nightly: - runs-on: [self-hosted, multi-test] + runs-on: [self-hosted, ci-gpu] steps: - name: Clean up previous run run: | diff --git a/docker/build_image.sh b/docker/build_image.sh index 1111f05816..b80068bca4 100755 --- a/docker/build_image.sh +++ b/docker/build_image.sh @@ -224,7 +224,7 @@ if [ "${BUILD_TYPE}" == "production" ]; then if [ "${MULTI}" == "true" ]; then DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ - --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform linux/amd64,linux/arm64 --target production-image ../ --push + --build-arg LOCAL_CHANGES="${LOCAL_CHANGES}" -t "${DOCKER_TAG}" --platform "${ARCH}" --target production-image ../ --push else DOCKER_BUILDKIT=1 docker buildx build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}"\ --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg REPO_URL="${REPO_URL}" --build-arg BUILD_FROM_SRC="${BUILD_FROM_SRC}"\ From fead9169a5d14b2f7a2ff70d737704c322d30d35 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sun, 15 Sep 2024 06:20:52 +0000 Subject: [PATCH 25/38] cache 3 --- docker/docker_nightly.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/docker_nightly.py b/docker/docker_nightly.py index a06b1184ea..434436f1db 100644 --- a/docker/docker_nightly.py +++ b/docker/docker_nightly.py @@ -39,7 +39,6 @@ cpp_dev_gpu_version = f"{project}:cpp-dev-gpu-{get_nightly_version()}" # Build Nightly images and append the date in the name - try_and_handle(f"./build_image.sh -m -n -t {organization}/{cpu_version}", dry_run) try_and_handle( f"./build_image.sh -g -cv cu121 -n -t {organization}/{gpu_version}", @@ -78,8 +77,6 @@ # ) # Push images with latest tag - # try_and_handle(f"docker push {organization}/{project}:latest-gpu", dry_run) - # try_and_handle( # f"docker push {organization}/{project}:latest-cpp-dev-cpu", dry_run # ) From 470b285fc6a6636f67d1e5cd126f77a970be1890 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sun, 15 Sep 2024 06:23:53 +0000 Subject: [PATCH 26/38] cache 4 --- docker/docker_nightly.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docker/docker_nightly.py b/docker/docker_nightly.py index 434436f1db..671d3eae19 100644 --- a/docker/docker_nightly.py +++ b/docker/docker_nightly.py @@ -40,10 +40,10 @@ # Build Nightly images and append the date in the name try_and_handle(f"./build_image.sh -m -n -t {organization}/{cpu_version}", dry_run) - try_and_handle( - f"./build_image.sh -g -cv cu121 -n -t {organization}/{gpu_version}", - dry_run, - ) + # try_and_handle( + # f"./build_image.sh -g -cv cu121 -n -t {organization}/{gpu_version}", + # dry_run, + # ) # try_and_handle( # f"./build_image.sh -bt dev -cpp -t {organization}/{cpp_dev_cpu_version}", # dry_run, @@ -54,7 +54,7 @@ # ) # Push Nightly images to official PyTorch Dockerhub account - try_and_handle(f"docker push {organization}/{gpu_version}", dry_run) + # try_and_handle(f"docker push {organization}/{gpu_version}", dry_run) # try_and_handle(f"docker push {organization}/{cpp_dev_cpu_version}", dry_run) # try_and_handle(f"docker push {organization}/{cpp_dev_gpu_version}", dry_run) @@ -63,10 +63,10 @@ f"docker buildx imagetools create --tag {organization}/{project}:latest-cpu {organization}/{cpu_version}", dry_run, ) - try_and_handle( - f"docker buildx imagetools create --tag {organization}/{project}:latest-gpu {organization}/{gpu_version}", - dry_run, - ) + # try_and_handle( + # f"docker buildx imagetools create --tag {organization}/{project}:latest-gpu {organization}/{gpu_version}", + # dry_run, + # ) # try_and_handle( # f"docker tag {organization}/{cpp_dev_cpu_version} {organization}/{project}:latest-cpp-dev-cpu", # dry_run, From 6bd6511e341f165d8b30b276b2790ad591c9070e Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sun, 15 Sep 2024 06:25:19 +0000 Subject: [PATCH 27/38] cache 4 --- .github/workflows/docker-nightly-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index 73ed1933eb..9f1fdb0eca 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -36,7 +36,7 @@ jobs: run: | cd docker sudo apt-get update - docker buildx use multibuilder + docker buildx create --name multibuilder --use --driver docker-container --bootstrap python docker_nightly.py --cleanup # - name: Push KServe Docker Nightly # run: | From df0a0ef8043eb595ad1f6bed24952b4da60ac9ed Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sun, 15 Sep 2024 06:49:09 +0000 Subject: [PATCH 28/38] final test --- .github/workflows/docker-nightly-build.yml | 2 +- docker/build_image.sh | 2 +- docker/docker_nightly.py | 62 ++++++++++------------ 3 files changed, 31 insertions(+), 35 deletions(-) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index 9f1fdb0eca..73ed1933eb 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -36,7 +36,7 @@ jobs: run: | cd docker sudo apt-get update - docker buildx create --name multibuilder --use --driver docker-container --bootstrap + docker buildx use multibuilder python docker_nightly.py --cleanup # - name: Push KServe Docker Nightly # run: | diff --git a/docker/build_image.sh b/docker/build_image.sh index b80068bca4..b5b9f8e87e 100755 --- a/docker/build_image.sh +++ b/docker/build_image.sh @@ -18,7 +18,7 @@ BUILD_NIGHTLY=false BUILD_FROM_SRC=false LOCAL_CHANGES=true PYTHON_VERSION=3.9 -ARCH="linux/arm64" +ARCH="linux/arm64,linux/amd64" MULTI=false for arg in "$@" diff --git a/docker/docker_nightly.py b/docker/docker_nightly.py index 671d3eae19..5f449b3e56 100644 --- a/docker/docker_nightly.py +++ b/docker/docker_nightly.py @@ -40,49 +40,45 @@ # Build Nightly images and append the date in the name try_and_handle(f"./build_image.sh -m -n -t {organization}/{cpu_version}", dry_run) - # try_and_handle( - # f"./build_image.sh -g -cv cu121 -n -t {organization}/{gpu_version}", - # dry_run, - # ) - # try_and_handle( - # f"./build_image.sh -bt dev -cpp -t {organization}/{cpp_dev_cpu_version}", - # dry_run, - # ) - # try_and_handle( - # f"./build_image.sh -bt dev -g -cv cu121 -cpp -t {organization}/{cpp_dev_gpu_version}", - # dry_run, - # ) + try_and_handle( + f"./build_image.sh -g -cv cu121 -n -t {organization}/{gpu_version}", + dry_run, + ) + try_and_handle( + f"./build_image.sh -bt dev -cpp -t {organization}/{cpp_dev_cpu_version}", + dry_run, + ) + try_and_handle( + f"./build_image.sh -bt dev -g -cv cu121 -cpp -t {organization}/{cpp_dev_gpu_version}", + dry_run, + ) # Push Nightly images to official PyTorch Dockerhub account - # try_and_handle(f"docker push {organization}/{gpu_version}", dry_run) - # try_and_handle(f"docker push {organization}/{cpp_dev_cpu_version}", dry_run) - # try_and_handle(f"docker push {organization}/{cpp_dev_gpu_version}", dry_run) + try_and_handle(f"docker push {organization}/{gpu_version}", dry_run) + try_and_handle(f"docker push {organization}/{cpp_dev_cpu_version}", dry_run) + try_and_handle(f"docker push {organization}/{cpp_dev_gpu_version}", dry_run) # Tag nightly images with latest try_and_handle( f"docker buildx imagetools create --tag {organization}/{project}:latest-cpu {organization}/{cpu_version}", dry_run, ) - # try_and_handle( - # f"docker buildx imagetools create --tag {organization}/{project}:latest-gpu {organization}/{gpu_version}", - # dry_run, - # ) - # try_and_handle( - # f"docker tag {organization}/{cpp_dev_cpu_version} {organization}/{project}:latest-cpp-dev-cpu", - # dry_run, - # ) - # try_and_handle( - # f"docker tag {organization}/{cpp_dev_gpu_version} {organization}/{project}:latest-cpp-dev-gpu", - # dry_run, - # ) + try_and_handle( + f"docker buildx imagetools create --tag {organization}/{project}:latest-gpu {organization}/{gpu_version}", + dry_run, + ) + try_and_handle( + f"docker tag {organization}/{cpp_dev_cpu_version} {organization}/{project}:latest-cpp-dev-cpu", + dry_run, + ) + try_and_handle( + f"docker tag {organization}/{cpp_dev_gpu_version} {organization}/{project}:latest-cpp-dev-gpu", + dry_run, + ) # Push images with latest tag - # try_and_handle( - # f"docker push {organization}/{project}:latest-cpp-dev-cpu", dry_run - # ) - # try_and_handle( - # f"docker push {organization}/{project}:latest-cpp-dev-gpu", dry_run - # ) + try_and_handle(f"docker push {organization}/{project}:latest-cpp-dev-cpu", dry_run) + try_and_handle(f"docker push {organization}/{project}:latest-cpp-dev-gpu", dry_run) # Cleanup built images if args.cleanup: From 473c05ed8337083ad07602c67efbde7e23d8ef20 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Sun, 15 Sep 2024 23:22:50 +0000 Subject: [PATCH 29/38] reverting temp changes --- .github/workflows/docker-nightly-build.yml | 36 ++++++++++------------ docker/docker_nightly.py | 2 +- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index 73ed1933eb..e988cc97c4 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -2,11 +2,9 @@ name: Push Docker Nightly on: # Run every day at 1:15pm - # schedule: - # - cron: "15 13 * * *" - # workflow_dispatch: - push: - branches: "docker_aarch" + schedule: + - cron: "15 13 * * *" + workflow_dispatch: jobs: nightly: @@ -30,23 +28,23 @@ jobs: submodules: recursive - name: Login to Docker env: - DOCKER_PASSWORD: ${{secrets.UDAI_TEST}} - run: docker login --username udaij12 --password "$DOCKER_PASSWORD" + DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} + run: docker login --username pytorchbot --password "$DOCKER_PASSWORD" - name: Push Docker Nightly run: | cd docker sudo apt-get update docker buildx use multibuilder python docker_nightly.py --cleanup - # - name: Push KServe Docker Nightly - # run: | - # cd kubernetes/kserve - # python docker_nightly.py --cleanup - # - name: Open issue on failure - # if: ${{ failure() && github.event_name == 'schedule' }} - # uses: dacbd/create-issue-action@v1 - # with: - # token: ${{ secrets.GITHUB_TOKEN }} - # title: Nightly Docker build failed - # body: Commit ${{ github.sha }} daily scheduled [CI run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) failed, please check why - # assignees: "" + - name: Push KServe Docker Nightly + run: | + cd kubernetes/kserve + python docker_nightly.py --cleanup + - name: Open issue on failure + if: ${{ failure() && github.event_name == 'schedule' }} + uses: dacbd/create-issue-action@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + title: Nightly Docker build failed + body: Commit ${{ github.sha }} daily scheduled [CI run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) failed, please check why + assignees: "" diff --git a/docker/docker_nightly.py b/docker/docker_nightly.py index 5f449b3e56..9d13e7b418 100644 --- a/docker/docker_nightly.py +++ b/docker/docker_nightly.py @@ -15,7 +15,7 @@ parser.add_argument( "--organization", type=str, - default="udaij12", + default="pytorch", help="The name of the Dockerhub organization where the images will be pushed", ) parser.add_argument( From d6b0f6391f533c33309a93054178cb2c6ab5d087 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Mon, 16 Sep 2024 19:16:48 +0000 Subject: [PATCH 30/38] testing official release --- .github/workflows/official_release_docker.yml | 25 +++++++++++-------- docker/build_upload_release.py | 25 +++++++++++++------ 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/.github/workflows/official_release_docker.yml b/.github/workflows/official_release_docker.yml index 63103eefa9..29205e9bef 100644 --- a/.github/workflows/official_release_docker.yml +++ b/.github/workflows/official_release_docker.yml @@ -1,16 +1,18 @@ name: Make an official Release of Docker Images on: - workflow_dispatch: - inputs: - upload_docker: - description: 'Upload TorchServe Docker Images?' - required: true - default: 'no' - upload_kfs: - description: 'Upload TorchServe-kfs Docker Images?' - required: true - default: 'no' + # workflow_dispatch: + # inputs: + # upload_docker: + # description: 'Upload TorchServe Docker Images?' + # required: true + # default: 'no' + # upload_kfs: + # description: 'Upload TorchServe-kfs Docker Images?' + # required: true + # default: 'no' + push: + -branches: "docker_aarch" jobs: official-release-docker: @@ -36,12 +38,13 @@ jobs: submodules: recursive - name: Login to Docker env: - DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} + DOCKER_PASSWORD: ${{secrets.UDAI_TEST}} run: docker login --username pytorchbot --password "$DOCKER_PASSWORD" - name: Build & Upload pytorch/torchserve Docker images if: github.event.inputs.upload_docker == 'yes' run: | cd docker + docker buildx use multibuilder python build_upload_release.py --cleanup - name: Build & Upload pytorch/torchserve-kfs Docker images if: github.event.inputs.upload_kfs == 'yes' diff --git a/docker/build_upload_release.py b/docker/build_upload_release.py index 72e33edec9..b4af680f52 100644 --- a/docker/build_upload_release.py +++ b/docker/build_upload_release.py @@ -13,7 +13,7 @@ parser.add_argument( "--organization", type=str, - default="pytorch", + default="udaij12", help="The name of the Dockerhub organization where the images will be pushed", ) parser.add_argument( @@ -31,7 +31,7 @@ organization = args.organization # Upload pytorch/torchserve docker binaries - try_and_handle(f"./build_image.sh -t {organization}/torchserve:latest", dry_run) + try_and_handle(f"./build_image.sh -m -t {organization}/torchserve:latest", dry_run) try_and_handle( f"./build_image.sh -g -cv cu121 -t {organization}/torchserve:latest-gpu", dry_run, @@ -44,14 +44,25 @@ f"./build_image.sh -bt dev -g -cv cu121 -cpp -t {organization}/torchserve:latest-cpp-dev-gpu", dry_run, ) + # try_and_handle( + # f"docker tag {organization}/torchserve:latest {organization}/torchserve:latest-cpu", + # dry_run, + # ) + try_and_handle( - f"docker tag {organization}/torchserve:latest {organization}/torchserve:latest-cpu", + f"docker buildx imagetools create --tag {organization}/torchserve:latest-cpu {organization}/torchserve:latest", dry_run, ) + try_and_handle( - f"docker tag {organization}/torchserve:latest {organization}/torchserve:{check_ts_version()}-cpu", + f"docker buildx imagetools create --tag {organization}/torchserve:{check_ts_version()}-cpu {organization}/torchserve:latest", dry_run, ) + + # try_and_handle( + # f"docker tag {organization}/torchserve:latest {organization}/torchserve:{check_ts_version()}-cpu", + # dry_run, + # ) try_and_handle( f"docker tag {organization}/torchserve:latest-gpu {organization}/torchserve:{check_ts_version()}-gpu", dry_run, @@ -66,12 +77,12 @@ ) for image in [ - f"{organization}/torchserve:latest", - f"{organization}/torchserve:latest-cpu", + # f"{organization}/torchserve:latest", + # f"{organization}/torchserve:latest-cpu", f"{organization}/torchserve:latest-gpu", f"{organization}/torchserve:latest-cpp-dev-cpu", f"{organization}/torchserve:latest-cpp-dev-gpu", - f"{organization}/torchserve:{check_ts_version()}-cpu", + # f"{organization}/torchserve:{check_ts_version()}-cpu", f"{organization}/torchserve:{check_ts_version()}-gpu", f"{organization}/torchserve:{check_ts_version()}-cpp-dev-cpu", f"{organization}/torchserve:{check_ts_version()}-cpp-dev-gpu", From e5edd18d2cc675218705ecbff1fa04d30837842a Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Mon, 16 Sep 2024 19:18:34 +0000 Subject: [PATCH 31/38] testing official release --- .github/workflows/official_release_docker.yml | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/official_release_docker.yml b/.github/workflows/official_release_docker.yml index 29205e9bef..cb72ef419a 100644 --- a/.github/workflows/official_release_docker.yml +++ b/.github/workflows/official_release_docker.yml @@ -1,18 +1,16 @@ name: Make an official Release of Docker Images on: - # workflow_dispatch: - # inputs: - # upload_docker: - # description: 'Upload TorchServe Docker Images?' - # required: true - # default: 'no' - # upload_kfs: - # description: 'Upload TorchServe-kfs Docker Images?' - # required: true - # default: 'no' - push: - -branches: "docker_aarch" + workflow_dispatch: + inputs: + upload_docker: + description: 'Upload TorchServe Docker Images?' + required: true + default: 'no' + upload_kfs: + description: 'Upload TorchServe-kfs Docker Images?' + required: true + default: 'no' jobs: official-release-docker: @@ -38,7 +36,7 @@ jobs: submodules: recursive - name: Login to Docker env: - DOCKER_PASSWORD: ${{secrets.UDAI_TEST}} + DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} run: docker login --username pytorchbot --password "$DOCKER_PASSWORD" - name: Build & Upload pytorch/torchserve Docker images if: github.event.inputs.upload_docker == 'yes' From ec9772fe8ca636a35619ee38a2046ca114e69603 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Mon, 16 Sep 2024 19:20:01 +0000 Subject: [PATCH 32/38] testing official release --- docker/build_upload_release.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/docker/build_upload_release.py b/docker/build_upload_release.py index b4af680f52..0f9cbb6505 100644 --- a/docker/build_upload_release.py +++ b/docker/build_upload_release.py @@ -13,7 +13,7 @@ parser.add_argument( "--organization", type=str, - default="udaij12", + default="pytorch", help="The name of the Dockerhub organization where the images will be pushed", ) parser.add_argument( @@ -44,10 +44,6 @@ f"./build_image.sh -bt dev -g -cv cu121 -cpp -t {organization}/torchserve:latest-cpp-dev-gpu", dry_run, ) - # try_and_handle( - # f"docker tag {organization}/torchserve:latest {organization}/torchserve:latest-cpu", - # dry_run, - # ) try_and_handle( f"docker buildx imagetools create --tag {organization}/torchserve:latest-cpu {organization}/torchserve:latest", @@ -59,10 +55,6 @@ dry_run, ) - # try_and_handle( - # f"docker tag {organization}/torchserve:latest {organization}/torchserve:{check_ts_version()}-cpu", - # dry_run, - # ) try_and_handle( f"docker tag {organization}/torchserve:latest-gpu {organization}/torchserve:{check_ts_version()}-gpu", dry_run, @@ -77,12 +69,9 @@ ) for image in [ - # f"{organization}/torchserve:latest", - # f"{organization}/torchserve:latest-cpu", f"{organization}/torchserve:latest-gpu", f"{organization}/torchserve:latest-cpp-dev-cpu", f"{organization}/torchserve:latest-cpp-dev-gpu", - # f"{organization}/torchserve:{check_ts_version()}-cpu", f"{organization}/torchserve:{check_ts_version()}-gpu", f"{organization}/torchserve:{check_ts_version()}-cpp-dev-cpu", f"{organization}/torchserve:{check_ts_version()}-cpp-dev-gpu", From 9af2db5e44af3c41ee22b87f792faa586cf4a65c Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Mon, 16 Sep 2024 19:43:24 +0000 Subject: [PATCH 33/38] adding kserve changes --- .github/workflows/official_release_docker.yml | 1 + kubernetes/kserve/build_image.sh | 12 +++++++++++- kubernetes/kserve/build_upload_release.py | 3 +-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/official_release_docker.yml b/.github/workflows/official_release_docker.yml index cb72ef419a..f3958101be 100644 --- a/.github/workflows/official_release_docker.yml +++ b/.github/workflows/official_release_docker.yml @@ -48,4 +48,5 @@ jobs: if: github.event.inputs.upload_kfs == 'yes' run: | cd kubernetes/kserve + docker buildx use multibuilder python build_upload_release.py --cleanup diff --git a/kubernetes/kserve/build_image.sh b/kubernetes/kserve/build_image.sh index b644dc37f7..b5df80267a 100755 --- a/kubernetes/kserve/build_image.sh +++ b/kubernetes/kserve/build_image.sh @@ -6,6 +6,8 @@ BASE_IMAGE="pytorch/torchserve:latest-cpu" DOCKER_FILE="Dockerfile" BUILD_NIGHTLY=false USE_CUSTOM_TAG=false +ARCH="linux/arm64,linux/amd64" +MULTI=false for arg in "$@" do @@ -38,6 +40,10 @@ do shift shift ;; + -m|--multi) + MULTI=true + shift + ;; esac done @@ -57,4 +63,8 @@ fi cp ../../frontend/server/src/main/resources/proto/*.proto . cp -r ../../third_party . -DOCKER_BUILDKIT=1 docker build --file "$DOCKER_FILE" --build-arg BASE_IMAGE=$BASE_IMAGE -t "$DOCKER_TAG" . +if [ "${MULTI}" == "true" ]; then + DOCKER_BUILDKIT=1 docker buildx build --file "$DOCKER_FILE" --build-arg BASE_IMAGE=$BASE_IMAGE --platform "${ARCH}" -t "$DOCKER_TAG" --push +else + DOCKER_BUILDKIT=1 docker buildx build --file "$DOCKER_FILE" --build-arg BASE_IMAGE=$BASE_IMAGE -t "$DOCKER_TAG" --load +fi diff --git a/kubernetes/kserve/build_upload_release.py b/kubernetes/kserve/build_upload_release.py index 55183c7a03..0c7c481671 100644 --- a/kubernetes/kserve/build_upload_release.py +++ b/kubernetes/kserve/build_upload_release.py @@ -31,7 +31,7 @@ organization = args.organization try_and_handle( - f"./build_image.sh -t {organization}/torchserve-kfs:{check_ts_version()}", + f"./build_image.sh -m -t {organization}/torchserve-kfs:{check_ts_version()}", dry_run, ) try_and_handle( @@ -40,7 +40,6 @@ ) for image in [ - f"{organization}/torchserve-kfs:{check_ts_version()}", f"{organization}/torchserve-kfs:{check_ts_version()}-gpu", ]: try_and_handle(f"docker push {image}", dry_run) From 385ba5599f514c2b57c11b286a528439523a7ed5 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Mon, 16 Sep 2024 22:40:23 +0000 Subject: [PATCH 34/38] kserve nightly --- .github/workflows/docker-nightly-build.yml | 1 + kubernetes/kserve/docker_nightly.py | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml index e988cc97c4..2bbd50712d 100644 --- a/.github/workflows/docker-nightly-build.yml +++ b/.github/workflows/docker-nightly-build.yml @@ -39,6 +39,7 @@ jobs: - name: Push KServe Docker Nightly run: | cd kubernetes/kserve + docker buildx use multibuilder python docker_nightly.py --cleanup - name: Open issue on failure if: ${{ failure() && github.event_name == 'schedule' }} diff --git a/kubernetes/kserve/docker_nightly.py b/kubernetes/kserve/docker_nightly.py index 209c0d976e..d5d3f13b76 100644 --- a/kubernetes/kserve/docker_nightly.py +++ b/kubernetes/kserve/docker_nightly.py @@ -37,19 +37,18 @@ gpu_version = f"{project}:gpu-{get_nightly_version()}" # Build Nightly images and append the date in the name - try_and_handle(f"./build_image.sh -n -t {organization}/{cpu_version}", dry_run) + try_and_handle(f"./build_image.sh -m -n -t {organization}/{cpu_version}", dry_run) try_and_handle( f"./build_image.sh -g -n -t {organization}/{gpu_version}", dry_run, ) # Push Nightly images to official PyTorch Dockerhub account - try_and_handle(f"docker push {organization}/{cpu_version}", dry_run) try_and_handle(f"docker push {organization}/{gpu_version}", dry_run) # Tag nightly images with latest try_and_handle( - f"docker tag {organization}/{cpu_version} {organization}/{project}:latest-cpu", + f"docker buildx imagetools create --tag {organization}/{project}:latest-cpu {organization}/{cpu_version}", dry_run, ) try_and_handle( @@ -58,7 +57,6 @@ ) # Push images with latest tag - try_and_handle(f"docker push {organization}/{project}:latest-cpu", dry_run) try_and_handle(f"docker push {organization}/{project}:latest-gpu", dry_run) # Cleanup built images From ebb0a6a72458346e129e701f64a840c10981d945 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Tue, 17 Sep 2024 17:37:57 +0000 Subject: [PATCH 35/38] adding build context --- kubernetes/kserve/build_image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/kserve/build_image.sh b/kubernetes/kserve/build_image.sh index b5df80267a..670915f1fc 100755 --- a/kubernetes/kserve/build_image.sh +++ b/kubernetes/kserve/build_image.sh @@ -64,7 +64,7 @@ cp ../../frontend/server/src/main/resources/proto/*.proto . cp -r ../../third_party . if [ "${MULTI}" == "true" ]; then - DOCKER_BUILDKIT=1 docker buildx build --file "$DOCKER_FILE" --build-arg BASE_IMAGE=$BASE_IMAGE --platform "${ARCH}" -t "$DOCKER_TAG" --push + DOCKER_BUILDKIT=1 docker buildx build --file "$DOCKER_FILE" --build-arg BASE_IMAGE=$BASE_IMAGE --platform "${ARCH}" -t "$DOCKER_TAG" --push . else - DOCKER_BUILDKIT=1 docker buildx build --file "$DOCKER_FILE" --build-arg BASE_IMAGE=$BASE_IMAGE -t "$DOCKER_TAG" --load + DOCKER_BUILDKIT=1 docker buildx build --file "$DOCKER_FILE" --build-arg BASE_IMAGE=$BASE_IMAGE -t "$DOCKER_TAG" --load . fi From 385ed7adcfaa327e35da63fa1b8a7524b440d5b7 Mon Sep 17 00:00:00 2001 From: udaij12 Date: Thu, 19 Sep 2024 15:26:56 -0700 Subject: [PATCH 36/38] changing nightly push --- kubernetes/kserve/build_image.sh | 2 +- kubernetes/kserve/docker_nightly.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/kubernetes/kserve/build_image.sh b/kubernetes/kserve/build_image.sh index 670915f1fc..39a005b8f0 100755 --- a/kubernetes/kserve/build_image.sh +++ b/kubernetes/kserve/build_image.sh @@ -66,5 +66,5 @@ cp -r ../../third_party . if [ "${MULTI}" == "true" ]; then DOCKER_BUILDKIT=1 docker buildx build --file "$DOCKER_FILE" --build-arg BASE_IMAGE=$BASE_IMAGE --platform "${ARCH}" -t "$DOCKER_TAG" --push . else - DOCKER_BUILDKIT=1 docker buildx build --file "$DOCKER_FILE" --build-arg BASE_IMAGE=$BASE_IMAGE -t "$DOCKER_TAG" --load . + DOCKER_BUILDKIT=1 docker buildx build --file "$DOCKER_FILE" --build-arg BASE_IMAGE=$BASE_IMAGE -t "$DOCKER_TAG" --push . fi diff --git a/kubernetes/kserve/docker_nightly.py b/kubernetes/kserve/docker_nightly.py index d5d3f13b76..3c329ddc89 100644 --- a/kubernetes/kserve/docker_nightly.py +++ b/kubernetes/kserve/docker_nightly.py @@ -51,14 +51,12 @@ f"docker buildx imagetools create --tag {organization}/{project}:latest-cpu {organization}/{cpu_version}", dry_run, ) + try_and_handle( - f"docker tag {organization}/{gpu_version} {organization}/{project}:latest-gpu", + f"docker buildx imagetools create --tag {organization}/{project}:latest-gpu {organization}/{gpu_version}", dry_run, ) - # Push images with latest tag - try_and_handle(f"docker push {organization}/{project}:latest-gpu", dry_run) - # Cleanup built images if args.cleanup: try_and_handle(f"docker system prune --all --volumes -f", dry_run) From a3079a4481085ff341f3acc249121265687e642a Mon Sep 17 00:00:00 2001 From: udaij12 Date: Fri, 20 Sep 2024 10:18:14 -0700 Subject: [PATCH 37/38] remove push on official --- kubernetes/kserve/build_upload_release.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/kubernetes/kserve/build_upload_release.py b/kubernetes/kserve/build_upload_release.py index 0c7c481671..e9dd311610 100644 --- a/kubernetes/kserve/build_upload_release.py +++ b/kubernetes/kserve/build_upload_release.py @@ -39,11 +39,6 @@ dry_run, ) - for image in [ - f"{organization}/torchserve-kfs:{check_ts_version()}-gpu", - ]: - try_and_handle(f"docker push {image}", dry_run) - # Cleanup built images if args.cleanup: try_and_handle(f"docker system prune --all --volumes -f", dry_run) From 093c40345940bb7675254e94966232b6d7d68a91 Mon Sep 17 00:00:00 2001 From: udaij12 Date: Fri, 20 Sep 2024 10:31:33 -0700 Subject: [PATCH 38/38] remove push on official --- kubernetes/kserve/docker_nightly.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/kubernetes/kserve/docker_nightly.py b/kubernetes/kserve/docker_nightly.py index 3c329ddc89..38c3bdea19 100644 --- a/kubernetes/kserve/docker_nightly.py +++ b/kubernetes/kserve/docker_nightly.py @@ -43,9 +43,6 @@ dry_run, ) - # Push Nightly images to official PyTorch Dockerhub account - try_and_handle(f"docker push {organization}/{gpu_version}", dry_run) - # Tag nightly images with latest try_and_handle( f"docker buildx imagetools create --tag {organization}/{project}:latest-cpu {organization}/{cpu_version}",