Mount public wpsoutputs data

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.32.0
+current_version = 1.33.0
 commit = True
 tag = False
 tag_name = {new_version}
@@ -30,11 +30,11 @@ search = {current_version}
 replace = {new_version}
 
 [bumpversion:file:RELEASE.txt]
-search = {current_version} 2023-09-22T16:09:24Z
+search = {current_version} 2023-09-25T13:02:37Z
 replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ}
 
 [bumpversion:part:releaseTime]
-values = 2023-09-22T16:09:24Z
+values = 2023-09-25T13:02:37Z
 
 [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template]
 search = 'version': '{current_version}'

.readthedocs.yml

@@ -19,14 +19,15 @@ sphinx:
 # Optionally build your docs in additional formats such as PDF and ePub
 formats: all
 
-# Optionally set the version of Python and requirements required to build your docs
+# Optionally set the Python requirements required to build your docs
 python:
-  version: 3.6
   install:
     - requirements: docs/requirements.txt
 
 #conda:
 #  environment: environment-docs.yml
 
 build:
-  image: stable
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.10"

CHANGES.md

@@ -17,6 +17,26 @@
 
 [//]: # (list changes here, using '-' for each new entry, remove this when items are added)
 
+[1.33.0](https://github.com/bird-house/birdhouse-deploy/tree/1.33.0) (2023-09-25)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+
+- Add public WPS outputs directory to Cowbird and add corresponding volume mount to JupyterHub.
+- Update `cowbird` service from [1.2.0](https://github.com/Ouranosinc/cowbird/tree/1.2.0)
+  to [2.1.0](https://github.com/Ouranosinc/cowbird/tree/2.1.0).
+- Require `MongoDB==5.0` Docker image for Cowbird's database.
+- Add `WPS_OUTPUTS_DIR` env variable to manage the location of the WPS outputs data.
+
+## Important
+Because of the new `MongoDB==5.0` database requirement for Cowbird that uses (potentially) distinct version from other 
+birds, a separate Docker image is employed only for Cowbird. If some processes, jobs, or other Cowbird-related data 
+was already defined on one of your server instances, manual transfer between the generic 
+`${DATA_PERSIST_ROOT}/mongodb_persist` to new  `${DATA_PERSIST_ROOT}/mongodb_cowbird_persist` directory must be 
+accomplished. The data in the new directory should then be migrated to the new version following the same procedure as
+described for Weaver in 
+[Database Migration](https://pavics-weaver.readthedocs.io/en/latest/installation.html?#database-migration).
+
 [1.32.0](https://github.com/bird-house/birdhouse-deploy/tree/1.32.0) (2023-09-22)
 ------------------------------------------------------------------------------------------------------------------
 

Makefile

@@ -1,7 +1,7 @@
 # Generic variables
 override SHELL       := bash
 override APP_NAME    := birdhouse-deploy
-override APP_VERSION := 1.32.0
+override APP_VERSION := 1.33.0
 
 # utility to remove comments after value of an option variable
 override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g")

README.rst

@@ -14,13 +14,13 @@ for a full-fledged production platform.
     * - releases
       - | |latest-version| |commits-since|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.32.0.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.33.0.svg
     :alt: Commits since latest release
-    :target: https://github.com/bird-house/birdhouse-deploy/compare/1.32.0...master
+    :target: https://github.com/bird-house/birdhouse-deploy/compare/1.33.0...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-1.32.0-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-1.33.0-blue.svg?style=flat
     :alt: Latest Tag
-    :target: https://github.com/bird-house/birdhouse-deploy/tree/1.32.0
+    :target: https://github.com/bird-house/birdhouse-deploy/tree/1.33.0
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
     :alt: ReadTheDocs Build Status (latest version)

RELEASE.txt

@@ -1 +1 @@
-1.32.0 2023-09-22T16:09:24Z
+1.33.0 2023-09-25T13:02:37Z

birdhouse/components/cowbird/config/cowbird/config.yml.template

@@ -32,6 +32,12 @@ handlers:
     priority: 1
     workspace_dir: ${WORKSPACE_DIR}
     jupyterhub_user_data_dir: ${JUPYTERHUB_USER_DATA_DIR}
+    wps_outputs_dir: ${WPS_OUTPUTS_DIR}
+    secure_data_proxy_name: ${SECURE_DATA_PROXY_NAME}
+    # wps_outputs_res_name: ${WPS_OUTPUTS_RES_NAME}
+    public_workspace_wps_outputs_subdir: ${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR}
+    # notebooks_dir_name: ${NOTEBOOKS_DIR_NAME}
+    # user_wps_outputs_dir_name: ${USER_WPS_OUTPUTS_DIR_NAME}
 
 # [Required] This section defines how to synchronize permissions between services when they share resources
 sync_permissions:

birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml

@@ -14,5 +14,6 @@ services:
   jupyterhub:
     environment:
       WORKSPACE_DIR: ${DATA_PERSIST_ROOT}/${USER_WORKSPACES}
+      PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR: ${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR}
     volumes:
       - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_ROOT}/${USER_WORKSPACES}"

birdhouse/components/cowbird/default.env

@@ -12,9 +12,12 @@
 # single quotes are important in below list to keep variable names intact until 'pavics-compose' parses them
 EXTRA_VARS='
     ${COWBIRD_VERSION}
-    ${COWBIRD_MONGODB_SERVICE}
+    ${COWBIRD_MONGODB_VERSION}
     ${COWBIRD_MONGODB_HOST}
     ${COWBIRD_MONGODB_PORT}
+    ${USER_WORKSPACES}
+    ${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR}
+    ${SECURE_DATA_PROXY_NAME}
 '
 # extend the original 'VARS' from 'birdhouse/pavics-compose.sh' to employ them for template substitution
 # adding them to 'VARS', they will also be validated in case of override of 'default.env' using 'env.local'
@@ -24,22 +27,32 @@ VARS="$VARS $EXTRA_VARS"
 # Cowbird Configuration
 # =====================
 
-export COWBIRD_VERSION="1.2.0"
+export COWBIRD_VERSION="2.1.0"
 
-# reuse the mongo instance provided by the core docker-compose
+# default release of the MongoDB version employed by Cowbird
+# note that MongoDB>=5.0 is REQUIRED for Cowbird>=2.0.0
+export COWBIRD_MONGODB_VERSION=5.0
+# either reuse the mongo instance provided by the core docker-compose
+# or use another service (e.g.: dedicated mongodb)
 # the 'host' is the mongo 'service' inter-docker network connection in this case
-# if another service should be used (e.g.: dedicated mongodb), adjust the variable
 # the selected 'mongo' service should be member of a network of same name
-export COWBIRD_MONGODB_SERVICE=mongodb
-export COWBIRD_MONGODB_HOST=${COWBIRD_MONGODB_SERVICE}
+export COWBIRD_MONGODB_HOST=cowbird-mongodb
 export COWBIRD_MONGODB_PORT=27017
 
 # Subdirectory containing the user workspaces used by Cowbird
 export USER_WORKSPACES="user_workspaces"
 
-# this dependency is only required if COWBIRD_MONGODB_SERVICE is the one provided in config/mongodb.
-# (include this by default to support backwards compatibility for now)
-COMPONENT_DEPENDENCIES="
-  $COMPONENT_DEPENDENCIES
-  ./config/mongodb
-"
+# Subdirectory containing the hardlinks to the public WPS outputs data
+# This directory will be mounted on the JupyterLab instances and is located by default
+# in the ${USER_WORKSPACES} directory.
+export PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR="public/wps_outputs"
+
+# Default name for the secure-data-proxy service from Magpie.
+export SECURE_DATA_PROXY_NAME="secure-data-proxy"
+
+# this dependency is only required if the mongo instance is the one provided in config/mongodb.
+# (include this for Cowbird<2.0.0)
+#COMPONENT_DEPENDENCIES="
+#  $COMPONENT_DEPENDENCIES
+#  ./config/mongodb
+#"

birdhouse/components/cowbird/docker-compose-extra.yml

@@ -23,20 +23,25 @@ services:
       # root user
       COWBIRD_FILESYSTEM_ADMIN_UID: 0
       COWBIRD_FILESYSTEM_ADMIN_GID: 0
-      WORKSPACE_DIR: /${USER_WORKSPACES}
+      PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR: ${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR}
+      SECURE_DATA_PROXY_NAME: ${SECURE_DATA_PROXY_NAME}
+      # Note that WPS_OUTPUTS_DIR and WORKSPACE_DIR must both point to paths from the same volume.
+      # This is to allow the creation of hardlinks between the wpsoutputs and the user workspace.
+      WPS_OUTPUTS_DIR: ${WPS_OUTPUTS_DIR}
+      WORKSPACE_DIR: ${DATA_PERSIST_ROOT}/${USER_WORKSPACES}
     links:
-      - "${COWBIRD_MONGODB_SERVICE}"
+      - cowbird-mongodb
     networks:
       - default
-      - "${COWBIRD_MONGODB_SERVICE}"
+      - cowbird-mongodb
     depends_on:
-      - "${COWBIRD_MONGODB_SERVICE}"
+      - cowbird-mongodb
     volumes:
       - ./components/cowbird/config/cowbird/config.yml:/opt/local/src/cowbird/config/cowbird.yml
       - ./components/cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini
       # even if not running tasks here, they must be registered to send them off to the right place!
       - ./components/cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py
-      - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:/${USER_WORKSPACES}"
+      - "${DATA_PERSIST_ROOT}:${DATA_PERSIST_ROOT}"
     restart: always
     logging: *default-logging
 
@@ -52,19 +57,33 @@ services:
       # root user
       COWBIRD_FILESYSTEM_ADMIN_UID: 0
       COWBIRD_FILESYSTEM_ADMIN_GID: 0
-      WORKSPACE_DIR: /${USER_WORKSPACES}
+      WORKSPACE_DIR: ${DATA_PERSIST_ROOT}/${USER_WORKSPACES}
     links:
-      - "${COWBIRD_MONGODB_SERVICE}"
+      - cowbird-mongodb
     networks:
       - default
-      - "${COWBIRD_MONGODB_SERVICE}"
+      - cowbird-mongodb
     depends_on:
-      - "${COWBIRD_MONGODB_SERVICE}"
+      - cowbird-mongodb
       - cowbird  # if not started first, sometimes celery misbehaves and will not pick jobs in queue
     volumes:
       - ./components/cowbird/config/cowbird/config.yml:/opt/local/src/cowbird/config/cowbird.yml
       - ./components/cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini
       - ./components/cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py
-      - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:/${USER_WORKSPACES}"
+      - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_ROOT}/${USER_WORKSPACES}"
     restart: always
     logging: *default-logging
+
+  # Dedicated database for Cowbird, since other 'mongodb' image does not employ the same version.
+  cowbird-mongodb:
+    image: mongo:${COWBIRD_MONGODB_VERSION}
+    container_name: cowbird-mongodb
+    networks:
+      - cowbird-mongodb
+    volumes:
+      - ${DATA_PERSIST_ROOT}/mongodb_cowbird_persist:/data/db
+    restart: always
+    logging: *default-logging
+
+networks:
+  cowbird-mongodb: {}

birdhouse/components/weaver/default.env

@@ -81,8 +81,8 @@ export WEAVER_WPS_PATH=/wps
 export WEAVER_WPS_OUTPUTS_PATH="/wpsoutputs/weaver"
 # data persistence of wps outputs (directory path)
 # NOTE:
-#   follow the same convension of 'wps_outputs/<bird>' as the rest of the stact
-export WEAVER_WPS_OUTPUTS_DIR='${DATA_PERSIST_ROOT}/wps_outputs/weaver'
+#   follow the same convention of 'wps_outputs/<bird>' as the rest of the stack
+export WEAVER_WPS_OUTPUTS_DIR='${WPS_OUTPUTS_DIR}/weaver'
 export WEAVER_WPS_WORKDIR="/tmp/wps_workdir/weaver"
 
 # logging

birdhouse/config/canarie-api/docker_configuration.py.template

@@ -109,8 +109,8 @@ SERVICES = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '1.32.0',
-            'releaseTime': '2023-09-22T16:09:24Z',
+            'version': '1.33.0',
+            'releaseTime': '2023-09-25T13:02:37Z',
             'institution': 'Ouranos',
             'researchSubject': 'Climatology',
             'supportEmail': '${SUPPORT_EMAIL}',
@@ -142,8 +142,8 @@ PLATFORMS = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '1.32.0',
-            'releaseTime': '2023-09-22T16:09:24Z',
+            'version': '1.33.0',
+            'releaseTime': '2023-09-25T13:02:37Z',
             'institution': 'Ouranos',
             'researchSubject': 'Climatology',
             'supportEmail': '${SUPPORT_EMAIL}',

birdhouse/config/jupyterhub/jupyterhub_config.py.template

@@ -88,6 +88,10 @@ if os.environ['WORKSPACE_DIR'] != jupyterhub_data_dir:
         "bind": join(jupyterhub_data_dir, "{username}"),
         "mode": "rw"
     }
+    c.DockerSpawner.volumes[join(os.environ['WORKSPACE_DIR'], os.environ['PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR'])] = {
+        "bind": join(notebook_dir, os.environ['PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR']),
+        "mode": "ro"
+    }
 
 container_gdrive_settings_path = join(container_home_dir, ".jupyter/lab/user-settings/@jupyterlab/google-drive/drive.jupyterlab-settings")
 host_gdrive_settings_path = os.environ['JUPYTER_GOOGLE_DRIVE_SETTINGS']

birdhouse/default.env

@@ -52,3 +52,5 @@ export DEFAULT_CONF_DIRS='
 
 export USER_WORKSPACE_UID=1000
 export USER_WORKSPACE_GID=1000
+
+export WPS_OUTPUTS_DIR="${DATA_PERSIST_ROOT}/wps_outputs"

docs/source/conf.py

@@ -69,9 +69,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.32.0'
+version = '1.33.0'
 # The full version, including alpha/beta/rc tags.
-release = '1.32.0'
+release = '1.33.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.