Move release scripts to their own 'release' dir

(and do some testing of the github action/scripts)

Signed-off-by: Tom Sellman <[email protected]>

.github/workflows/build.yml

@@ -252,6 +252,8 @@ jobs:
     if: ${{ contains(needs.build.outputs.commit_message,'[release]') }}
     runs-on: ubuntu-latest
     needs: build
+    permissions:
+        contents: write
     timeout-minutes: 10
     steps:
       # setup steps
@@ -290,7 +292,7 @@ jobs:
 
       # release step
       - name: Release
-        run: bash .github/scripts/release.sh
+        run: bash release/main.sh
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DEPLOY_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_DEPLOY_SECRET_ACCESS_KEY }}
@@ -301,7 +303,7 @@ jobs:
           MAVEN_PUBLISH_URL: ${{ vars.MAVEN_PLUGINS_PUBLISH_URL }}
           PLUGINS_INDEX_JSON: ${{ vars.PLUGINS_INDEX_JSON }}
           S3_RELEASE_BUCKET: ${{ vars.S3_RELEASE_BUCKET }}
-          SEQERA_REGISTRY: ${{ vars.SEQERA_PUBLIC_CR_URL }}
+          SEQERA_CONTAINER_REGISTRY: ${{ vars.SEQERA_PUBLIC_CR_URL }}
 
       # upload steps
       - name: Upload artifacts (libs)

Makefile

@@ -98,8 +98,9 @@ pack:
 	BUILD_PACK=1 ./gradlew pack
 
 #
-# Upload final package to GitHub
+# Initiate the nextflow release process
 #
+.PHONY: release
 release:
 	./make-release.sh
 

build.gradle

@@ -279,8 +279,7 @@ ext.enableSignArchives = project.findProperty('signing.keyId')
 ext.coreProjects = projects( ':nextflow', ':nf-commons', ':nf-httpfs' )
 
 /*
- * Maven central deployment
- * http://central.sonatype.org/pages/gradle.html
+ * Maven deployment
  */
 configure(coreProjects) {
     apply plugin: 'maven-publish'
@@ -343,8 +342,8 @@ configure(coreProjects) {
 
         repositories {
             maven {
-                url = System.getenv('MAVEN_PUBLISH_URL')
-                    ?: ( version.endsWith('-SNAPSHOT') ? "s3://maven.seqera.io/snapshots" : "s3://maven.seqera.io/releases" )
+                url = System.getenv('MAVEN_PUBLISH_URL') ?: 's3://maven.seqera.io'
+                url = version.endsWith('-SNAPSHOT') ? "$url/snapshots" : "$url/releases"
 
                 credentials(AwsCredentials) {
                     accessKey = System.getenv('AWS_ACCESS_KEY_ID') ?: findProperty('aws_access_key_id')

make-release.sh

@@ -1,6 +1,14 @@
 #!/usr/bin/env bash
 set -e
 
+# -----------------------------------------------------------------------------
+# Nextflow release entrypoint
+#
+# This script starts a Nextflow release, and is executed by the `make release`
+# command. It will help guide you through the manual steps required to perform
+# a release, and then trigger a release build on the CI system (Github Actions).
+# -----------------------------------------------------------------------------
+
 cd "$(dirname "$0")"
 
 # read the nextflow version

plugins/build.gradle

@@ -135,8 +135,8 @@ subprojects {
         }
         repositories {
             maven {
-                url = System.getenv('MAVEN_PUBLISH_URL')
-                    ?: ( version.endsWith('-SNAPSHOT') ? "s3://maven.seqera.io/snapshots" : "s3://maven.seqera.io/releases" )
+                url = System.getenv('MAVEN_PUBLISH_URL') ?: 's3://maven.seqera.io'
+                url = version.endsWith('-SNAPSHOT') ? "$url/snapshots" : "$url/releases"
 
                 credentials(AwsCredentials) {
                     accessKey = System.getenv('AWS_ACCESS_KEY_ID') ?: findProperty('aws_access_key_id')

.github/scripts/deploy-plugins-to-github.sh → release/deploy-plugins-to-github.sh

@@ -2,7 +2,7 @@
 set -e
 
 # change to the project root
-cd "$(dirname "$0")/../.."
+cd "$(dirname "$0")/.."
 
 GH_ORG=${GH_ORG:-'nextflow-io'}
 
@@ -14,7 +14,11 @@ function get_plugin_version() {
 }
 
 # deploy plugin artifacts to github releases
-echo "Publishing plugins to github"
+echo "
+----------------------------------
+-- Publishing plugins to github --
+----------------------------------
+"
 
 for plugin in plugins/nf-*; do
   if [[ -d "$plugin" ]]; then
@@ -24,11 +28,12 @@ for plugin in plugins/nf-*; do
     plugin_version=$(get_plugin_version "$plugin")
 
     # check if release already exists
+    release_exists=false
     gh release view --repo "$plugin_repo" "$plugin_version" > /dev/null 2>&1 \
       && release_exists=true
 
     # if not exists, create github release, with zip & meta json files
-    if [[ $release_exists ]]; then
+    if [[ $release_exists == true ]]; then
       echo "Plugin $plugin_name $plugin_version already deployed to github, skipping"
     else
       gh release create \

.github/scripts/deploy-plugins-to-maven.sh → release/deploy-plugins-to-maven.sh

@@ -2,9 +2,13 @@
 set -e
 
 # change to the project root
-cd "$(dirname "$0")/../.."
+cd "$(dirname "$0")/.."
 
-echo "Publishing plugins to maven"
+echo "
+---------------------------------
+-- Publishing plugins to maven --
+---------------------------------
+"
 
 # the release process should have already built the jars, so to avoid re-compiling everything
 # we can tell gradle to skip all non publish/publication related tasks

release/deploy-to-docker.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -e
+
+# change to the project root
+cd "$(dirname "$0")/.."
+
+# read the nextflow version
+read -r NF_VERSION<VERSION
+
+# build the docker image
+echo "
+---------------------------------------
+-- Building nextflow container image --
+---------------------------------------
+"
+cd docker
+make build version="$NF_VERSION"
+
+# push to docker hub
+echo "
+-------------------------------------
+-- Pushing container to docker hub --
+-------------------------------------
+"
+IMAGE_TAG="nextflow/nextflow:$NF_VERSION"
+docker push "$IMAGE_TAG"
+
+# push to seqera registry
+echo "
+------------------------------------------
+-- Pushing container to seqera registry --
+------------------------------------------
+"
+if [ -z "$SEQERA_CONTAINER_REGISTRY" ]; then
+  echo "Missing env var SEQERA_CONTAINER_REGISTRY"
+  exit 1
+fi
+SEQERA_IMAGE_TAG="$SEQERA_CONTAINER_REGISTRY/$IMAGE_TAG"
+
+docker tag "$IMAGE_TAG" "$SEQERA_IMAGE_TAG"
+docker push "$SEQERA_IMAGE_TAG"
+
+echo "Done"

.github/scripts/deploy-to-github.sh → release/deploy-to-github.sh

@@ -2,7 +2,7 @@
 set -e
 
 # change to the project root
-cd "$(dirname "$0")/../.."
+cd "$(dirname "$0")/.."
 
 # read the nextflow version
 read -r NF_VERSION<VERSION
@@ -31,7 +31,11 @@ function get_change_notes() {
   done < changelog.txt
 }
 
-echo "Publishing nextflow release to github"
+echo "
+-------------------------------------------
+-- Publishing nextflow release to github --
+-------------------------------------------
+"
 
 # create a github (pre)release and attach launcher and dist files
 # use --verify-tag to fail if tag doesn't exist

.github/scripts/deploy-to-maven.sh → release/deploy-to-maven.sh

@@ -2,9 +2,13 @@
 set -e
 
 # change to the project root
-cd "$(dirname "$0")/../.."
+cd "$(dirname "$0")/.."
 
-echo "Publishing nextflow jars to maven"
+echo "
+---------------------------------------
+-- Publishing nextflow jars to maven --
+---------------------------------------
+"
 
 # the release process should have already built the jars, so to avoid re-compiling everything
 # we can tell gradle to skip all non publish/publication related tasks

.github/scripts/deploy-to-s3.sh → release/deploy-to-s3.sh

@@ -2,19 +2,26 @@
 set -e
 
 # change to the project root
-cd "$(dirname "$0")/../.."
+cd "$(dirname "$0")/.."
 
 # read the nextflow version
 read -r NF_VERSION<VERSION
 
+echo "
+-------------------------------------------
+-- Uploading nextflow distribution to S3 --
+-------------------------------------------
+"
+
 S3_RELEASE_BUCKET=${S3_RELEASE_BUCKET:-'www2.nextflow.io'}
 S3_RELEASE_DIR="releases/v$NF_VERSION"
 
 # check if the release already exists
+release_exists=false
 aws s3api head-object --bucket "$S3_RELEASE_BUCKET" --key "$S3_RELEASE_DIR/nextflow" > /dev/null 2>&1 \
   && release_exists=true
 
-if [[ $release_exists ]]; then
+if [[ $release_exists == true ]]; then
   echo "Version $NF_VERSION already deployed to S3, skipping"
   exit
 fi

release/main.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -e
+
+# -----------------------------------------------------------------------------
+# Nextflow release script
+#
+# This is the orchestration script for Nextflow releases.
+#
+# It is intended to be run by 'headless' CI environments (eg Github Actions)
+# to execute a release.You probably don't want to run this script directly.
+#
+# Instead, use the `make release` command to be guided through the process.
+# -----------------------------------------------------------------------------
+
+# set defaults for unspecified env vars
+export GH_ORG=${GH_ORG:-'nextflow-io'}
+export GH_USER=${GH_USER:-'pditommaso'} # TODO - use a service user for releases
+export GH_USER_EMAIL=${GH_USER_EMAIL:-'[email protected]'}
+
+export MAVEN_PUBLISH_URL=${MAVEN_PUBLISH_URL:-'s3://maven.seqera.io'}
+export PLUGINS_INDEX_JSON=${PLUGINS_INDEX_JSON:-'https://github.com/nextflow-io/plugins/main/plugins.json'}
+export S3_RELEASE_BUCKET=${S3_RELEASE_BUCKET:-'www2.nextflow.io'}
+export SEQERA_CONTAINER_REGISTRY=${SEQERA_CONTAINER_REGISTRY:-'public.cr.seqera.io'}
+
+# change to the project root
+cd "$(dirname "$0")/.."
+
+# build artifacts
+make distribution
+
+# tag release
+./release/tag-release.sh
+
+# deploy to maven
+./release/deploy-to-maven.sh
+
+# deploy to S3
+./release/deploy-to-s3.sh
+
+# deploy to docker
+./release/deploy-to-docker.sh
+
+# deploy to github
+./release/deploy-to-github.sh
+
+# deploy plugins
+./release/deploy-plugins-to-maven.sh
+./release/deploy-plugins-to-github.sh
+./release/update-plugins-index.sh
+
+# finally, publish the new launcher
+./release/publish-launcher-script.sh