feat: extension management #76
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Gubbins extension Tests | |
| on: | |
| push: | |
| paths-ignore: | |
| - 'packages/diracx-web/**' | |
| branches-ignore: | |
| - dependabot/** | |
| - release-please-** | |
| pull_request: | |
| branches-ignore: | |
| - release-please-** | |
| jobs: | |
| # Run lint and type checks | |
| run-lint: | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'push' || github.repository == 'DIRACGrid/diracx-web' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 'lts/*' | |
| cache: 'npm' | |
| - name: Install dependencies | |
| run: npm ci | |
| # ========================================================================# | |
| # WARNING: In your CI/CD pipeline, you should remove the following step # | |
| # ------------------------------------------------------------------------# | |
| - name: Build Library | |
| run: npm run build:diracx-web-components | |
| # ------------------------------------------------------------------------# | |
| # ========================================================================# | |
| - name: Check code-quality rules | |
| run: npm --prefix packages/extensions run lint | |
| - name: Check typescript type validity | |
| run: npm --prefix packages/extensions run ts-lint | |
| # ============================================================================# | |
| # WARNING: In your CI/CD pipeline, you should remove the following jobs # | |
| # ----------------------------------------------------------------------------# | |
| # By default, the gubbins extension is part of a monorepo and uses a local # | |
| # version of `diracx-web-components`. This default setup is not # | |
| # representative of a standalone extension configuration. Here we want to # | |
| # closely mimic a standalone extension. To do so, we: # | |
| # - Isolate the packages/extensions directory: Move its content to a new # | |
| # repository or directory outside the monorepo. # | |
| # - Update Configuration: Adjust relevant variables to align with a # | |
| # standalone setup. # | |
| prepare-gubbins-backend: | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event_name != 'push' || github.repository == 'DIRACGrid/diracx-web' }} | |
| defaults: | |
| run: | |
| # We need extglob for REFERENCE_BRANCH substitution | |
| shell: bash -l -O extglob {0} | |
| steps: | |
| - name: Clone source | |
| run: | | |
| cd .. | |
| git clone https://github.com/DIRACGrid/diracx.git | |
| # Prepare the gubbins extension | |
| - name: Where the magic happens (Move extensions to a temporary directory) | |
| run: | | |
| # We have to copy the code to another directory | |
| # and make it a git repository by itself because otherwise the | |
| # root in the pyproject to do not make sense once mounted | |
| # in the containers. | |
| cp -r ../diracx/extensions/gubbins /tmp/ | |
| sed -i 's@../..@.@g' /tmp/gubbins/pyproject.toml | |
| sed -i 's@../../@@g' /tmp/gubbins/gubbins-*/pyproject.toml | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gubbins | |
| path: /tmp/gubbins | |
| include-hidden-files: true | |
| # Prepare the gubbins image | |
| # - Build the gubbins wheels | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Installing dependencies | |
| run: | | |
| cd ../diracx | |
| python -m pip install \ | |
| build \ | |
| python-dateutil \ | |
| pytz \ | |
| readme_renderer[md] \ | |
| requests \ | |
| setuptools_scm | |
| - name: Build distributions | |
| run: | | |
| cd ../diracx | |
| for pkg_dir in $PWD/diracx-*; do | |
| echo "Building $pkg_dir" | |
| python -m build --outdir $PWD/dist $pkg_dir | |
| done | |
| # Also build the diracx metapackage | |
| python -m build --outdir $PWD/dist . | |
| # And build the gubbins package | |
| for pkg_dir in $PWD/extensions/gubbins/gubbins-*; do | |
| # Skip the testing package | |
| if [[ "${pkg_dir}" =~ .*testing.* ]]; | |
| then | |
| echo "Do not build ${pkg_dir}"; | |
| continue; | |
| fi | |
| echo "Building $pkg_dir" | |
| python -m build --outdir $PWD/dist $pkg_dir | |
| done | |
| - name: "Find wheels" | |
| id: find_wheel | |
| run: | | |
| cd ../diracx/dist | |
| # We need to copy them there to be able to access them in the RUN --mount | |
| cp diracx*.whl gubbins*.whl ../extensions/containers/services/ | |
| for wheel_fn in *.whl; do | |
| pkg_name=$(basename "${wheel_fn}" | cut -d '-' -f 1) | |
| echo "${pkg_name}-wheel-name=$(ls "${pkg_name}"-*.whl)" >> $GITHUB_OUTPUT | |
| done | |
| # - Build the gubbins image using the wheels | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build and export service | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ../diracx/extensions/containers/services | |
| tags: gubbins/services:dev | |
| outputs: type=docker,dest=/tmp/gubbins_services_image.tar | |
| build-args: | | |
| EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration | |
| EXTENSION_CUSTOM_SOURCES_TO_INSTALL=/bindmount/gubbins_db*.whl,/bindmount/gubbins_routers*.whl,/bindmount/gubbins_client*.whl | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gubbins-services-img | |
| path: /tmp/gubbins_services_image.tar | |
| prepare-gubbins-frontend-as-a-standalone: | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event_name != 'push' || github.repository == 'DIRACGrid/diracx-web' }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 'lts/*' | |
| registry-url: 'https://registry.npmjs.org' | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Build diracx-web-components | |
| working-directory: packages/diracx-web-components | |
| run: | | |
| npm pack | |
| - name: Where the magic happens (Move extensions to a temporary directory) | |
| run: | | |
| # Move the extensions to the temporary directory | |
| cp -r packages/extensions /tmp/gubbins-web | |
| # Move the library package to the extensions directory | |
| mv packages/diracx-web-components/*.tgz /tmp/gubbins-web/diracx-web-components.tgz | |
| # Update the package.json to point to the local diracx-web-components | |
| jq '.dependencies["@dirac-grid/diracx-web-components"] = "file:./diracx-web-components.tgz"' /tmp/gubbins-web/package.json > /tmp/gubbins-web/package.temp.json | |
| mv /tmp/gubbins-web/package.temp.json /tmp/gubbins-web/package.json | |
| # Update the package.json to point to the correct postinstall script | |
| jq '.scripts.postinstall = "node ./node_modules/@axa-fr/react-oidc/bin/copy-service-worker-files.mjs public"' /tmp/gubbins-web/package.json > /tmp/gubbins-web/package.temp.json | |
| mv /tmp/gubbins-web/package.temp.json /tmp/gubbins-web/package.json | |
| # Update the next.config.js to prevent the standalone extension from searching a local version of diracx-web-components that does not exist | |
| echo 'const nextConfig = { output: "export", images: { unoptimized: true }}; export default nextConfig;' > /tmp/gubbins-web/next.config.js | |
| - name: Install extensions dependencies | |
| working-directory: /tmp/gubbins-web | |
| run: npm install | |
| - name: Upload the extensions directory | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gubbins-web | |
| path: /tmp/gubbins-web | |
| retention-days: 1 | |
| # ----------------------------------------------------------------------------# | |
| # In the following jobs, you should: # | |
| # - Replace /tmp/gubbins-web with the path to your extension source code # | |
| # - Remove the `needs: prepare-gubbins-as-a-standalone` line # | |
| # ============================================================================# | |
| # Build the extension image | |
| build-gubbins-dev-image: | |
| runs-on: ubuntu-latest | |
| needs: prepare-gubbins-frontend-as-a-standalone | |
| if: ${{ github.event_name != 'push' || github.repository == 'DIRACGrid/diracx-web' }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to GitHub container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| # ========================================================================# | |
| # WARNING: In your CI/CD pipeline, you should remove the following steps # | |
| # ------------------------------------------------------------------------# | |
| - name: Download gubbins-web | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gubbins-web | |
| path: /tmp/gubbins-web | |
| # ------------------------------------------------------------------------# | |
| # ========================================================================# | |
| # In your extension, you have to push the image to the registry (check deployment.yml) | |
| - name: Build Docker image (extension example) | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: /tmp/gubbins-web | |
| push: false | |
| tags: ghcr.io/diracgrid/diracx-web/gubbins:dev | |
| platforms: linux/amd64,linux/arm64 | |
| # Run the demo with the extension source code (frontend and backend) | |
| run-demo: | |
| runs-on: ubuntu-latest | |
| needs: [prepare-gubbins-frontend-as-a-standalone, prepare-gubbins-backend] | |
| if: github.event_name != 'push' || github.repository == 'DIRACGrid/diracx-web' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Clone source | |
| run: | | |
| cd .. | |
| git clone https://github.com/DIRACGrid/diracx-charts.git | |
| # ========================================================================# | |
| # WARNING: In your CI/CD pipeline, you should remove the following steps # | |
| # ------------------------------------------------------------------------# | |
| - name: Download gubbins | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gubbins | |
| path: /tmp/gubbins | |
| - name: Create a new git repository | |
| run: | | |
| git init /tmp/gubbins/ | |
| - name: Clone diracx source | |
| run: | | |
| cd .. | |
| git clone https://github.com/DIRACGrid/diracx.git | |
| - name: Download gubbins:services image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gubbins-services-img | |
| path: /tmp/ | |
| - name: Load docker image | |
| run: docker load --input /tmp/gubbins_services_image.tar | |
| - name: Download gubbins-web | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gubbins-web | |
| path: /tmp/gubbins-web | |
| # ------------------------------------------------------------------------# | |
| # ========================================================================# | |
| # Runs the demo with the extension source code | |
| - name: Start demo | |
| run: | | |
| cd .. | |
| diracx-charts/run_demo.sh --exit-when-done --set-value developer.autoReload=false --ci-values ./diracx/extensions/gubbins_values.yaml --load-docker-image "gubbins/services:dev" diracx/ /tmp/gubbins/ /tmp/gubbins-web/ | |
| - name: Debugging information | |
| run: | | |
| cd ../diracx-charts | |
| export KUBECONFIG=$PWD/.demo/kube.conf | |
| .demo/kubectl get pods | |
| for pod_name in $(.demo/kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do | |
| echo "${pod_name}" | |
| .demo/kubectl describe pod/"${pod_name}" || true | |
| for container_name in $(.demo/kubectl get pods $pod_name -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}'); do | |
| echo $pod_name $container_name | |
| .demo/kubectl logs "${pod_name}" -c "${container_name}" || true | |
| done | |
| done | |
| - name: Check for success | |
| run: | | |
| cd ../diracx-charts | |
| if [ ! -f ".demo/.success" ]; then | |
| echo "Demo failed" | |
| cat ".demo/.failed" | |
| exit 1 | |
| fi | |
| # Find the URL to be tested and put it in the github environment for the next step | |
| - name: Set DIRACX_URL | |
| run: echo "DIRACX_URL=https://$(ifconfig | grep 'inet ' | awk '{ print $2 }' | grep -v '^127' | head -n 1 | cut -d '/' -f 1).nip.io:8000" >> $GITHUB_ENV | |
| # Run e2e tests with Cypress | |
| - name: Start Cypress | |
| uses: cypress-io/github-action@v6 | |
| with: | |
| browser: chrome | |
| config: baseUrl=${{ env.DIRACX_URL }} | |
| project: /tmp/gubbins-web |