diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cf1889f7f..cb845535b 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.32.0 +current_version = 1.33.0 commit = True tag = False tag_name = {new_version} @@ -30,11 +30,11 @@ search = {current_version} replace = {new_version} [bumpversion:file:RELEASE.txt] -search = {current_version} 2023-09-22T16:09:24Z +search = {current_version} 2023-09-25T13:02:37Z replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ} [bumpversion:part:releaseTime] -values = 2023-09-22T16:09:24Z +values = 2023-09-25T13:02:37Z [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template] search = 'version': '{current_version}' diff --git a/.readthedocs.yml b/.readthedocs.yml index 54b6f6fc9..1798af9e9 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -19,9 +19,8 @@ sphinx: # Optionally build your docs in additional formats such as PDF and ePub formats: all -# Optionally set the version of Python and requirements required to build your docs +# Optionally set the Python requirements required to build your docs python: - version: 3.6 install: - requirements: docs/requirements.txt @@ -29,4 +28,6 @@ python: # environment: environment-docs.yml build: - image: stable + os: "ubuntu-22.04" + tools: + python: "3.10" diff --git a/CHANGES.md b/CHANGES.md index 9460b7931..da76d242b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,26 @@ [//]: # (list changes here, using '-' for each new entry, remove this when items are added) +[1.33.0](https://github.com/bird-house/birdhouse-deploy/tree/1.33.0) (2023-09-25) +------------------------------------------------------------------------------------------------------------------ + +## Changes + +- Add public WPS outputs directory to Cowbird and add corresponding volume mount to JupyterHub. +- Update `cowbird` service from [1.2.0](https://github.com/Ouranosinc/cowbird/tree/1.2.0) + to [2.1.0](https://github.com/Ouranosinc/cowbird/tree/2.1.0). +- Require `MongoDB==5.0` Docker image for Cowbird's database. +- Add `WPS_OUTPUTS_DIR` env variable to manage the location of the WPS outputs data. + +## Important +Because of the new `MongoDB==5.0` database requirement for Cowbird that uses (potentially) distinct version from other +birds, a separate Docker image is employed only for Cowbird. If some processes, jobs, or other Cowbird-related data +was already defined on one of your server instances, manual transfer between the generic +`${DATA_PERSIST_ROOT}/mongodb_persist` to new `${DATA_PERSIST_ROOT}/mongodb_cowbird_persist` directory must be +accomplished. The data in the new directory should then be migrated to the new version following the same procedure as +described for Weaver in +[Database Migration](https://pavics-weaver.readthedocs.io/en/latest/installation.html?#database-migration). + [1.32.0](https://github.com/bird-house/birdhouse-deploy/tree/1.32.0) (2023-09-22) ------------------------------------------------------------------------------------------------------------------ diff --git a/Makefile b/Makefile index f7c9ef7e6..cd37a2b7b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Generic variables override SHELL := bash override APP_NAME := birdhouse-deploy -override APP_VERSION := 1.32.0 +override APP_VERSION := 1.33.0 # utility to remove comments after value of an option variable override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g") diff --git a/README.rst b/README.rst index fd93acba8..e0d9ddd47 100644 --- a/README.rst +++ b/README.rst @@ -14,13 +14,13 @@ for a full-fledged production platform. * - releases - | |latest-version| |commits-since| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.32.0.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.33.0.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/1.32.0...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/1.33.0...master -.. |latest-version| image:: https://img.shields.io/badge/tag-1.32.0-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-1.33.0-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/1.32.0 + :target: https://github.com/bird-house/birdhouse-deploy/tree/1.33.0 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version) diff --git a/RELEASE.txt b/RELEASE.txt index a352691e0..99985762c 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -1 +1 @@ -1.32.0 2023-09-22T16:09:24Z +1.33.0 2023-09-25T13:02:37Z diff --git a/birdhouse/components/cowbird/config/cowbird/config.yml.template b/birdhouse/components/cowbird/config/cowbird/config.yml.template index 9e32de893..2ea78e562 100644 --- a/birdhouse/components/cowbird/config/cowbird/config.yml.template +++ b/birdhouse/components/cowbird/config/cowbird/config.yml.template @@ -32,6 +32,12 @@ handlers: priority: 1 workspace_dir: ${WORKSPACE_DIR} jupyterhub_user_data_dir: ${JUPYTERHUB_USER_DATA_DIR} + wps_outputs_dir: ${WPS_OUTPUTS_DIR} + secure_data_proxy_name: ${SECURE_DATA_PROXY_NAME} + # wps_outputs_res_name: ${WPS_OUTPUTS_RES_NAME} + public_workspace_wps_outputs_subdir: ${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR} + # notebooks_dir_name: ${NOTEBOOKS_DIR_NAME} + # user_wps_outputs_dir_name: ${USER_WPS_OUTPUTS_DIR_NAME} # [Required] This section defines how to synchronize permissions between services when they share resources sync_permissions: diff --git a/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml b/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml index 04942fdec..af5307d01 100644 --- a/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml +++ b/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml @@ -14,5 +14,6 @@ services: jupyterhub: environment: WORKSPACE_DIR: ${DATA_PERSIST_ROOT}/${USER_WORKSPACES} + PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR: ${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR} volumes: - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_ROOT}/${USER_WORKSPACES}" diff --git a/birdhouse/components/cowbird/default.env b/birdhouse/components/cowbird/default.env index 3fb07dd6a..70bd64714 100644 --- a/birdhouse/components/cowbird/default.env +++ b/birdhouse/components/cowbird/default.env @@ -12,9 +12,12 @@ # single quotes are important in below list to keep variable names intact until 'pavics-compose' parses them EXTRA_VARS=' ${COWBIRD_VERSION} - ${COWBIRD_MONGODB_SERVICE} + ${COWBIRD_MONGODB_VERSION} ${COWBIRD_MONGODB_HOST} ${COWBIRD_MONGODB_PORT} + ${USER_WORKSPACES} + ${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR} + ${SECURE_DATA_PROXY_NAME} ' # extend the original 'VARS' from 'birdhouse/pavics-compose.sh' to employ them for template substitution # adding them to 'VARS', they will also be validated in case of override of 'default.env' using 'env.local' @@ -24,22 +27,32 @@ VARS="$VARS $EXTRA_VARS" # Cowbird Configuration # ===================== -export COWBIRD_VERSION="1.2.0" +export COWBIRD_VERSION="2.1.0" -# reuse the mongo instance provided by the core docker-compose +# default release of the MongoDB version employed by Cowbird +# note that MongoDB>=5.0 is REQUIRED for Cowbird>=2.0.0 +export COWBIRD_MONGODB_VERSION=5.0 +# either reuse the mongo instance provided by the core docker-compose +# or use another service (e.g.: dedicated mongodb) # the 'host' is the mongo 'service' inter-docker network connection in this case -# if another service should be used (e.g.: dedicated mongodb), adjust the variable # the selected 'mongo' service should be member of a network of same name -export COWBIRD_MONGODB_SERVICE=mongodb -export COWBIRD_MONGODB_HOST=${COWBIRD_MONGODB_SERVICE} +export COWBIRD_MONGODB_HOST=cowbird-mongodb export COWBIRD_MONGODB_PORT=27017 # Subdirectory containing the user workspaces used by Cowbird export USER_WORKSPACES="user_workspaces" -# this dependency is only required if COWBIRD_MONGODB_SERVICE is the one provided in config/mongodb. -# (include this by default to support backwards compatibility for now) -COMPONENT_DEPENDENCIES=" - $COMPONENT_DEPENDENCIES - ./config/mongodb -" +# Subdirectory containing the hardlinks to the public WPS outputs data +# This directory will be mounted on the JupyterLab instances and is located by default +# in the ${USER_WORKSPACES} directory. +export PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR="public/wps_outputs" + +# Default name for the secure-data-proxy service from Magpie. +export SECURE_DATA_PROXY_NAME="secure-data-proxy" + +# this dependency is only required if the mongo instance is the one provided in config/mongodb. +# (include this for Cowbird<2.0.0) +#COMPONENT_DEPENDENCIES=" +# $COMPONENT_DEPENDENCIES +# ./config/mongodb +#" diff --git a/birdhouse/components/cowbird/docker-compose-extra.yml b/birdhouse/components/cowbird/docker-compose-extra.yml index 0f1d837f4..4a905b878 100644 --- a/birdhouse/components/cowbird/docker-compose-extra.yml +++ b/birdhouse/components/cowbird/docker-compose-extra.yml @@ -23,20 +23,25 @@ services: # root user COWBIRD_FILESYSTEM_ADMIN_UID: 0 COWBIRD_FILESYSTEM_ADMIN_GID: 0 - WORKSPACE_DIR: /${USER_WORKSPACES} + PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR: ${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR} + SECURE_DATA_PROXY_NAME: ${SECURE_DATA_PROXY_NAME} + # Note that WPS_OUTPUTS_DIR and WORKSPACE_DIR must both point to paths from the same volume. + # This is to allow the creation of hardlinks between the wpsoutputs and the user workspace. + WPS_OUTPUTS_DIR: ${WPS_OUTPUTS_DIR} + WORKSPACE_DIR: ${DATA_PERSIST_ROOT}/${USER_WORKSPACES} links: - - "${COWBIRD_MONGODB_SERVICE}" + - cowbird-mongodb networks: - default - - "${COWBIRD_MONGODB_SERVICE}" + - cowbird-mongodb depends_on: - - "${COWBIRD_MONGODB_SERVICE}" + - cowbird-mongodb volumes: - ./components/cowbird/config/cowbird/config.yml:/opt/local/src/cowbird/config/cowbird.yml - ./components/cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini # even if not running tasks here, they must be registered to send them off to the right place! - ./components/cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py - - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:/${USER_WORKSPACES}" + - "${DATA_PERSIST_ROOT}:${DATA_PERSIST_ROOT}" restart: always logging: *default-logging @@ -52,19 +57,33 @@ services: # root user COWBIRD_FILESYSTEM_ADMIN_UID: 0 COWBIRD_FILESYSTEM_ADMIN_GID: 0 - WORKSPACE_DIR: /${USER_WORKSPACES} + WORKSPACE_DIR: ${DATA_PERSIST_ROOT}/${USER_WORKSPACES} links: - - "${COWBIRD_MONGODB_SERVICE}" + - cowbird-mongodb networks: - default - - "${COWBIRD_MONGODB_SERVICE}" + - cowbird-mongodb depends_on: - - "${COWBIRD_MONGODB_SERVICE}" + - cowbird-mongodb - cowbird # if not started first, sometimes celery misbehaves and will not pick jobs in queue volumes: - ./components/cowbird/config/cowbird/config.yml:/opt/local/src/cowbird/config/cowbird.yml - ./components/cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini - ./components/cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py - - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:/${USER_WORKSPACES}" + - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_ROOT}/${USER_WORKSPACES}" restart: always logging: *default-logging + + # Dedicated database for Cowbird, since other 'mongodb' image does not employ the same version. + cowbird-mongodb: + image: mongo:${COWBIRD_MONGODB_VERSION} + container_name: cowbird-mongodb + networks: + - cowbird-mongodb + volumes: + - ${DATA_PERSIST_ROOT}/mongodb_cowbird_persist:/data/db + restart: always + logging: *default-logging + +networks: + cowbird-mongodb: {} diff --git a/birdhouse/components/weaver/default.env b/birdhouse/components/weaver/default.env index bb3e66835..fd58cf85b 100644 --- a/birdhouse/components/weaver/default.env +++ b/birdhouse/components/weaver/default.env @@ -81,8 +81,8 @@ export WEAVER_WPS_PATH=/wps export WEAVER_WPS_OUTPUTS_PATH="/wpsoutputs/weaver" # data persistence of wps outputs (directory path) # NOTE: -# follow the same convension of 'wps_outputs/' as the rest of the stact -export WEAVER_WPS_OUTPUTS_DIR='${DATA_PERSIST_ROOT}/wps_outputs/weaver' +# follow the same convention of 'wps_outputs/' as the rest of the stack +export WEAVER_WPS_OUTPUTS_DIR='${WPS_OUTPUTS_DIR}/weaver' export WEAVER_WPS_WORKDIR="/tmp/wps_workdir/weaver" # logging diff --git a/birdhouse/config/canarie-api/docker_configuration.py.template b/birdhouse/config/canarie-api/docker_configuration.py.template index 0c89368d0..892f76ded 100644 --- a/birdhouse/config/canarie-api/docker_configuration.py.template +++ b/birdhouse/config/canarie-api/docker_configuration.py.template @@ -109,8 +109,8 @@ SERVICES = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.32.0', - 'releaseTime': '2023-09-22T16:09:24Z', + 'version': '1.33.0', + 'releaseTime': '2023-09-25T13:02:37Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', @@ -142,8 +142,8 @@ PLATFORMS = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.32.0', - 'releaseTime': '2023-09-22T16:09:24Z', + 'version': '1.33.0', + 'releaseTime': '2023-09-25T13:02:37Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', diff --git a/birdhouse/config/jupyterhub/jupyterhub_config.py.template b/birdhouse/config/jupyterhub/jupyterhub_config.py.template index 4eb0ec5e3..e3f6d5ae7 100644 --- a/birdhouse/config/jupyterhub/jupyterhub_config.py.template +++ b/birdhouse/config/jupyterhub/jupyterhub_config.py.template @@ -88,6 +88,10 @@ if os.environ['WORKSPACE_DIR'] != jupyterhub_data_dir: "bind": join(jupyterhub_data_dir, "{username}"), "mode": "rw" } + c.DockerSpawner.volumes[join(os.environ['WORKSPACE_DIR'], os.environ['PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR'])] = { + "bind": join(notebook_dir, os.environ['PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR']), + "mode": "ro" + } container_gdrive_settings_path = join(container_home_dir, ".jupyter/lab/user-settings/@jupyterlab/google-drive/drive.jupyterlab-settings") host_gdrive_settings_path = os.environ['JUPYTER_GOOGLE_DRIVE_SETTINGS'] diff --git a/birdhouse/default.env b/birdhouse/default.env index f5f082d03..3504d84c0 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -52,3 +52,5 @@ export DEFAULT_CONF_DIRS=' export USER_WORKSPACE_UID=1000 export USER_WORKSPACE_GID=1000 + +export WPS_OUTPUTS_DIR="${DATA_PERSIST_ROOT}/wps_outputs" diff --git a/docs/source/conf.py b/docs/source/conf.py index 8b1daae3f..122121fcc 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,9 +69,9 @@ # built documents. # # The short X.Y version. -version = '1.32.0' +version = '1.33.0' # The full version, including alpha/beta/rc tags. -release = '1.32.0' +release = '1.33.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages.