diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 53632616a..9b1d3f8d2 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.6.5 +current_version = 2.7.0 commit = True tag = False tag_name = {new_version} @@ -30,11 +30,11 @@ search = {current_version} replace = {new_version} [bumpversion:file:RELEASE.txt] -search = {current_version} 2024-12-18T18:55:30Z +search = {current_version} 2024-12-19T13:52:44Z replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ} [bumpversion:part:releaseTime] -values = 2024-12-18T18:55:30Z +values = 2024-12-19T13:52:44Z [bumpversion:file(version):birdhouse/components/canarie-api/docker_configuration.py.template] search = 'version': '{current_version}' diff --git a/.gitignore b/.gitignore index 128ede050..16f981110 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ venv/ ## Testing .pytest_cache/ *.log +birdhouse/data/ \ No newline at end of file diff --git a/CHANGES.md b/CHANGES.md index d937e8616..7be494e32 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,53 @@ [//]: # (list changes here, using '-' for each new entry, remove this when items are added) +[2.7.0](https://github.com/bird-house/birdhouse-deploy/tree/2.7.0) (2024-12-19) +------------------------------------------------------------------------------------------------------------------ + +## Changes + +- Enable local deployment to improve development and testing + + The Birdhouse stack can now be deployed locally and accessed on a browser on the host machine without the need + for an SSL certificate. This is useful for local development and for running tests against the full stack while developing and in CI environments. + + To enable this, add the new `optional-components/local-dev-test` component to `BIRDHOUSE_EXTRA_CONF_DIRS` and + set the following environment variables in the local environment file: + + * `export BIRDHOUSE_FQDN=host.docker.internal` + * `export BIRDHOUSE_HTTP_ONLY=True` + + You should also add ``host.docker.internal`` to your ``/etc/hosts`` file pointing to the loopback address so + that URLs generated by Birdhouse that refer to ``host.docker.internal`` will resolve properly in a browser: + + ``` + echo '127.0.0.1 host.docker.internal' | sudo tee -a /etc/hosts + ``` + + After deploying the stack, you can now interact with the Birdhouse software at ``http://host.docker.internal`` + from the machine that is the docker host. + + In order to implement the changes above, the following non-breaking changes have been made to the deployment code: + + - added a configuration variable `BIRDHOUSE_HTTP_ONLY` which is not set by default. If set to `True` the `proxy` component will only serve content over `http` (not `https`). + - added the following configuration variables. These should not be set directly unless you really know what you're doing: + - `BIRDHOUSE_PROXY_SCHEME`: default remains `https`. If `BIRDHOUSE_HTTP_ONLY` is `True` then the default becomes `http` + - `PROXY_INCLUDE_HTTPS`: default remains `include /etc/nginx/conf.d/https.include;`. If `BIRDHOUSE_HTTP_ONLY` is `True`, the default is that the variable is unset. + - changed the default values for the following configuration variables: + - `BIRDHOUSE_ALLOW_UNSECURE_HTTP`: default remains `""`. If `BIRDHOUSE_HTTP_ONLY` is `True` then the default becomes `True`. + - logs are written to stderr by default. Previously they were written to stdout. + - this allows us to call scripts and programmatically use their outputs. Previously log entries would need to be + manually filtered out before program outputs could be used. + - added the `--log-stdout` and `--log-file` flags to the `bin/birdhouse` interface to allow redirecting logs to + stdout or to a specific file instead. + - log redirection can also now be set using environment variables: + - `BIRDHOUSE_LOG_FD` can be used to redirect logs to a file descriptor (ex: `BIRDHOUSE_LOG_FD=3`) + - `BIRDHOUSE_LOG_FILE` can be used to redirect logs to file (ex: `BIRDHOUSE_LOG_FILE=/some/file/on/disk.log`) + - Note that the variables here should not be set in the local environment file since that file is sourced **after** + some logs are written. Instead, set these by exporting them in the parent process that calls `bin/birdhouse`. + - for backwards compatibility, if scripts are not called through the `bin/birdhouse` interface, logs will still be + written to stdout. + [2.6.5](https://github.com/bird-house/birdhouse-deploy/tree/2.6.5) (2024-12-18) ------------------------------------------------------------------------------------------------------------------ diff --git a/Makefile b/Makefile index 02137e9be..edfc59cc2 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Generic variables override SHELL := bash override APP_NAME := birdhouse-deploy -override APP_VERSION := 2.6.5 +override APP_VERSION := 2.7.0 # utility to remove comments after value of an option variable override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g") diff --git a/README.rst b/README.rst index 4e2992935..32eafe845 100644 --- a/README.rst +++ b/README.rst @@ -18,13 +18,13 @@ for a full-fledged production platform. * - citation - | |citation| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.6.5.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.7.0.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/2.6.5...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/2.7.0...master -.. |latest-version| image:: https://img.shields.io/badge/tag-2.6.5-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-2.7.0-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/2.6.5 + :target: https://github.com/bird-house/birdhouse-deploy/tree/2.7.0 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version) diff --git a/RELEASE.txt b/RELEASE.txt index 4b2ffe05e..8f3a1ebae 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -1 +1 @@ -2.6.5 2024-12-18T18:55:30Z +2.7.0 2024-12-19T13:52:44Z diff --git a/bin/birdhouse b/bin/birdhouse index 659bc01e9..be1bcfa71 100755 --- a/bin/birdhouse +++ b/bin/birdhouse @@ -2,12 +2,23 @@ THIS_FILE="$(readlink -f "$0" || realpath "$0")" THIS_DIR="$(dirname "${THIS_FILE}")" +THIS_BASENAME="$(basename "${THIS_FILE}")" COMPOSE_DIR="$(dirname "${THIS_DIR}")/birdhouse" export BIRDHOUSE_COMPOSE="${BIRDHOUSE_COMPOSE:-"${COMPOSE_DIR}/birdhouse-compose.sh"}" export __BIRDHOUSE_SUPPORTED_INTERFACE=True -USAGE="USAGE: $0 [-h|--help] [-b|--backwards-compatible] [-e|--env-file local-env-file] {info|compose|configs}" +USAGE="USAGE: $THIS_BASENAME [-h|--help] + [-b|--backwards-compatible] + [-e|--env-file local-env-file] + [-q|--quiet] [-q|--quiet {DEBUG|INFO|WARN|ERROR|CRITICAL}] + [-s|--log-stdout] [-s|--log-stdout {DEBUG|INFO|WARN|ERROR|CRITICAL}] + [-l|--log-file log-file-path] + [-l|--log-file {DEBUG|INFO|WARN|ERROR|CRITICAL} log-file-path] + [-L|--log-level {DEBUG|INFO|WARN|ERROR|CRITICAL}] + {info|compose|configs}" +USAGE=$(echo $USAGE | tr "\n" " ") + HELP="$USAGE Manage the Birdhouse software stack. @@ -18,12 +29,19 @@ Commands: configs Load or execute commands in the Birdhouse configuration environment Options: - -h, --help Print this message and exit - -b, --backwards-compatible Run in backwards compatible mode - -e, --env-file string Override the local environment file, default is ${COMPOSE_DIR}/env.local + -h, --help Print this message and exit + -b, --backwards-compatible Run in backwards compatible mode + -e, --env-file string Override the local environment file, default is ${COMPOSE_DIR}/env.local + -s, --log-stdout Write logs to stdout for all log levels, default is to write to stderr + -s, --log-stdout {DEBUG|INFO|WARN|ERROR|CRITICAL} Write logs to stdout for the given log level only (this option can be repeated) + -l, --log-file path Write logs to this file path for all log levels + -l, --log-file {DEBUG|INFO|WARN|ERROR|CRITICAL} path Write logs to this file path for the given log level only (this option can be repeated), this takes precedence over the --log-file option for all log levels + -q, --quiet Do not write logs to stdout or stderr for all log levels. Logs will still be written to a file if --log-file is set + -q, --quiet {DEBUG|INFO|WARN|ERROR|CRITICAL} Do not write logs to stdout or stderr for the given log level only (this option can be repeated), Logs will still be written to a file if --log-file is set + -L, --log-level {DEBUG|INFO|WARN|ERROR} Set log level, default is INFO " -CONFIGS_USAGE="USAGE: $0 configs [-h|--help] [-d|--default] {[-p|--print-config-command] | [-c|--command command]}" +CONFIGS_USAGE="USAGE: $THIS_BASENAME configs [-h|--help] [-d|--default] {[-p|--print-config-command] | [-c|--command command]}" CONFIGS_HELP="$CONFIGS_USAGE Load or execute commands in the Birdhouse configuration environment. @@ -32,15 +50,16 @@ Options: -d, --default Only load/print a command for the default configuration settings, not those specified by the local environment file -p, --print-config-command Print a command that can be used to load configuration settings as environment variables -c, --command string Execute the given command after loading configuration settings - -q, --quiet Suppress stdout when loading configuration settings for the '--command' option. +Deprecated Options: + -q, --quiet Suppress stdout when loading configuration settings for the '--command' option. [DEPRECATED: use the --quiet option directly under birdhouse instead] Example Usage: - $ ${0} configs -c 'echo \${BIRDHOUSE_FQDN}' + $ ${THIS_BASENAME} configs -c 'echo \${BIRDHOUSE_FQDN}' example.com # This is the value of BIRDHOUSE_FQDN as determined by the current configuration settings - $ ${0} configs -p + $ ${THIS_BASENAME} configs -p . /path/to/configs/file/to/source && read_configs - $ eval \$(${0} configs) + $ eval \$(${THIS_BASENAME} configs) $ echo \${BIRDHOUSE_FQDN} example.com # This is the value of BIRDHOUSE_FQDN as determined by the current configuration settings " @@ -148,7 +167,7 @@ parse_configs_args() { print_config_command elif [ "${CONFIGS_CMD+set}" = 'set' ]; then if [ "${CONFIGS_QUIET}" = "True" ]; then - eval "$(print_config_command)" > /dev/null + eval "$(print_config_command)" 2> /dev/null else eval "$(print_config_command)" fi @@ -164,6 +183,15 @@ parse_configs_args() { esac } +# Echos "True" if the first argument is a valid log level +check_log_dest_override() { + case "$1" in + DEBUG|INFO|WARN|ERROR|CRITICAL) + echo True + ;; + esac +} + # Parse arguments and options parse_args() { case "$1" in @@ -194,6 +222,54 @@ parse_args() { shift parse_args "$@" ;; + -q|--quiet) + shift + if [ "$(check_log_dest_override "$1")" ]; then + export BIRDHOUSE_LOG_DEST_OVERRIDE="${BIRDHOUSE_LOG_DEST_OVERRIDE}:$1:quiet:" + shift + else + export BIRDHOUSE_LOG_QUIET=True # The argument here takes precedence over the env variable + fi + parse_args "$@" + ;; + -s|--log-stdout) + shift + if [ "$(check_log_dest_override "$1")" ]; then + export BIRDHOUSE_LOG_DEST_OVERRIDE="${BIRDHOUSE_LOG_DEST_OVERRIDE}:$1:fd:1" + shift + else + export BIRDHOUSE_LOG_FD=1 # The argument here takes precedence over the env variable + fi + parse_args "$@" + ;; + -l=*|--log-file=*) + arg_value="${1#*=}" + shift + parse_args --log-file "${arg_value}" "$@" + ;; + -l|--log-file) + shift + # Note: cannot log to a file named DEBUG, INFO, WARN, ERROR, or CRITICAL + if [ "$(check_log_dest_override "$1")" ]; then + export BIRDHOUSE_LOG_DEST_OVERRIDE="${BIRDHOUSE_LOG_DEST_OVERRIDE}:$1:file:$(realpath -- "$2")" + shift + else + export BIRDHOUSE_LOG_FILE=$(realpath -- "$1") # The argument here takes precedence over the env variable + fi + shift + parse_args "$@" + ;; + -L=*|--log-level=*) + arg_value="${1#*=}" + shift + parse_args --log-level "${arg_value}" "$@" + ;; + -L|--log-level) + shift + export BIRDHOUSE_LOG_LEVEL="$1" # The argument here takes precedence over the env variable + shift + parse_args "$@" + ;; info) shift "${BIRDHOUSE_COMPOSE}" info "$@" @@ -212,7 +288,7 @@ parse_args() { echo "$HELP" ;; -??*) - parse_multiple_short_flags parse_configs_args "$@" + parse_multiple_short_flags parse_args "$@" ;; *) >&2 echo "$USAGE" diff --git a/birdhouse/README.rst b/birdhouse/README.rst index d79d30254..b2a021150 100644 --- a/birdhouse/README.rst +++ b/birdhouse/README.rst @@ -291,6 +291,57 @@ Starting and managing the lifecycle of the VM: # not needed normally during tight development loop vagrant provision +Deploy locally for development or test purposes +----------------------------------------------- + +If you are developing this code base or want to test out a new feature locally on a machine, you may want to deploy +the Birdhouse stack locally. + +There are two strategies available to deploy the Birdhouse stack locally: + +- `Use HTTP scheme deployment`_ +- `Use a Self-Signed SSL certificate`_ + +Use HTTP scheme deployment +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To deploy locally, enable the :ref:`local-dev-test` component. Also set the following two variables in your local +environment file: + +- ``export BIRDHOUSE_FQDN=host.docker.internal`` +- ``export BIRDHOUSE_HTTP_ONLY=True`` + +This will allow you to access the Birdhouse software in a browser on your local machine using +the URL ``http://host.docker.internal`` without the need for an SSL certificate or to expose ports 80 and 443 +publicly. + +Use a Self-Signed SSL certificate +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The `Use HTTP scheme deployment`_ strategy described above will send all information over ``http`` instead of using +``https``. + +If there are any features that you want to test locally using ``https``, you can deploy locally using a self-signed +SSL certificate. + +You may also need to add the following to the ``docker compose`` settings for the ``twitcher`` component if you're +not able to access protected URLs: + +.. code:: yaml + + services: + twitcher: + environment: + REQUESTS_CA_BUNDLE: "${BIRDHOUSE_SSL_CERTIFICATE}" + volumes: + - "${BIRDHOUSE_SSL_CERTIFICATE}:${BIRDHOUSE_SSL_CERTIFICATE}:ro" + + +.. warning:: + + Self-signed certificates are not fully supported by the components of the Birdhouse stack and some features may + not be fully functional when self-signed certificates are enabled. For example, accessing other components through + the JupyterLab interface may fail with an ``SSLError``. Framework tests --------------- diff --git a/birdhouse/birdhouse-compose.sh b/birdhouse/birdhouse-compose.sh index 04475a12b..0731e97c8 100755 --- a/birdhouse/birdhouse-compose.sh +++ b/birdhouse/birdhouse-compose.sh @@ -105,8 +105,8 @@ fi create_compose_conf_list # this sets COMPOSE_CONF_LIST log INFO "Displaying resolved compose configurations:" -echo "COMPOSE_CONF_LIST=" -echo ${COMPOSE_CONF_LIST} | tr ' ' '\n' | grep -v '^-f' +log INFO "COMPOSE_CONF_LIST=" +log INFO ${COMPOSE_CONF_LIST} | tr ' ' '\n' | grep -v '^-f' if [ x"$1" = x"info" ]; then log INFO "Stopping before execution of docker-compose command." @@ -123,12 +123,17 @@ if [ x"$1" = x"up" ]; then log INFO "Executing '$COMPONENT_PRE_COMPOSE_UP'" sh ${SHELL_EXEC_FLAGS} "$COMPONENT_PRE_COMPOSE_UP" fi + COMPONENT_PRE_COMPOSE_UP_INCLUDE="$adir/pre-docker-compose-up.include" + if [ -f "$COMPONENT_PRE_COMPOSE_UP_INCLUDE" ]; then + log INFO "Sourcing '$COMPONENT_PRE_COMPOSE_UP_INCLUDE'" + . "$COMPONENT_PRE_COMPOSE_UP_INCLUDE" + fi done fi log INFO "Executing docker-compose with extra options: $* ${COMPOSE_EXTRA_OPTS}" -# the PROXY_SECURE_PORT is a little trick to make the compose file invalid without the usage of this wrapper script -PROXY_SECURE_PORT=443 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} $* ${COMPOSE_EXTRA_OPTS} +# the PROXY_HTTP_PORT is a little trick to make the compose file invalid without the usage of this wrapper script +PROXY_HTTP_PORT=80 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} $* ${COMPOSE_EXTRA_OPTS} ERR=$? if [ ${ERR} -gt 0 ]; then log ERROR "docker-compose error, exit code ${ERR}" @@ -148,11 +153,11 @@ while [ $# -gt 0 ] do if [ x"$1" = x"up" ]; then # we restart the proxy after an up to make sure nginx continue to work if any container IP address changes - PROXY_SECURE_PORT=443 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} restart proxy + PROXY_HTTP_PORT=80 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} restart proxy # run postgres post-startup setup script # Note: this must run before the post-docker-compose-up scripts since some may expect postgres databases to exist - postgres_id=$(PROXY_SECURE_PORT=443 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} ps -q postgres 2> /dev/null) + postgres_id=$(PROXY_HTTP_PORT=80 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} ps -q postgres 2> /dev/null) if [ ! -z "$postgres_id" ]; then docker exec ${postgres_id} /postgres-setup.sh fi diff --git a/birdhouse/components/README.rst b/birdhouse/components/README.rst index c9c1ffebe..0adb96657 100644 --- a/birdhouse/components/README.rst +++ b/birdhouse/components/README.rst @@ -540,7 +540,7 @@ exposed by the current stack instance. Once this component is enabled, STAC API ``https:///stac-browser`` endpoint. In order to make the STAC browser the default entrypoint, define the following in the ``env.local`` file:: - export BIRDHOUSE_PROXY_ROOT_LOCATION="return 302 https://\$host/stac-browser;" + export BIRDHOUSE_PROXY_ROOT_LOCATION='return 302 ${BIRDHOUSE_PROXY_SCHEME}://\$host/stac-browser;' Here is a sample search query using a CLI:: @@ -567,7 +567,7 @@ An endpoint monitoring tool that shows the current status of other components in Usage ----- -The service is available at ``https://${BIRDHOUSE_FQDN_PUBLIC}/canarie`` +The service is available at ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/canarie`` How to Enable the Component --------------------------- @@ -599,7 +599,7 @@ degree-days of cooling, the duration of heatwaves, etc. This returns annual valu Usage ----- -The service is available at ``https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch`` +The service is available at ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch`` How to Enable the Component --------------------------- @@ -618,7 +618,7 @@ Geospatial Web. Usage ----- -The service is available at ``https://${BIRDHOUSE_FQDN_PUBLIC}/geoserver``. For usage and +The service is available at ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/geoserver``. For usage and configuration options please refer to the `Geoserver documentation`_. .. _Geoserver documentation: https://docs.geoserver.org @@ -637,7 +637,7 @@ A Web Processing Service for compliance checks used in the climate science commu Usage ----- -The service is available at ``https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird`` +The service is available at ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird`` How to Enable the Component --------------------------- @@ -654,7 +654,7 @@ end-users. Usage ----- -The service is available at ``https://${BIRDHOUSE_FQDN_PUBLIC}/jupyter``. Users are able to log in to Jupyterhub using the +The service is available at ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/jupyter``. Users are able to log in to Jupyterhub using the same user name and password as Magpie. They will then be able to launch a personal jupyterlab server. How to Enable the Component @@ -673,7 +673,7 @@ User/Group/Service/Resource/Permission management and integrates with Twitcher. Usage ----- -The service is available at ``https://${BIRDHOUSE_FQDN_PUBLIC}/magpie``. For usage and configuration options please +The service is available at ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/magpie``. For usage and configuration options please refer to the `Magpie documentation`_. .. _Magpie documentation: https://pavics-magpie.readthedocs.io @@ -706,7 +706,7 @@ A web based container deployment and management tool. Usage ----- -The service is available at ``https://${BIRDHOUSE_FQDN_PUBLIC}/portainer/``. For usage and configuration options please +The service is available at ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/portainer/``. For usage and configuration options please refer to the `portainer documentation`_. How to Enable the Component @@ -757,7 +757,7 @@ processing as well as time series analysis. Usage ----- -The service is available at ``https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven`` +The service is available at ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven`` How to Enable the Component --------------------------- @@ -775,7 +775,7 @@ Climate Data Catalog and Format Renderers. See the `Thredds documentation`_ for Usage ----- -The catalog is available at the ``https://${BIRDHOUSE_FQDN_PUBLIC}/thredds`` endpoint. +The catalog is available at the ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/thredds`` endpoint. How to Enable the Component --------------------------- @@ -811,7 +811,7 @@ of all processes executed by these services. Usage ----- -All outputs from these processes will become available at the ``https://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs`` endpoint. +All outputs from these processes will become available at the ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs`` endpoint. By default, this endpoint is not protected. To secure access to this endpoint it is highly recommended to enable the `./optional-components/secure-data-proxy` component as well. diff --git a/birdhouse/components/canarie-api/docker_configuration.py.template b/birdhouse/components/canarie-api/docker_configuration.py.template index 04d718db3..f937d7392 100644 --- a/birdhouse/components/canarie-api/docker_configuration.py.template +++ b/birdhouse/components/canarie-api/docker_configuration.py.template @@ -10,7 +10,7 @@ import requests_cache # see entrypoint script logger = logging.getLogger("canarie-api-config") -MY_SERVER_NAME = 'https://${BIRDHOUSE_FQDN_PUBLIC}/canarie' +MY_SERVER_NAME = '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/canarie' DATABASE = { 'filename': '/data/stats.db', @@ -108,8 +108,8 @@ SERVICES = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '2.6.5', - 'releaseTime': '2024-12-18T18:55:30Z', + 'version': '2.7.0', + 'releaseTime': '2024-12-19T13:52:44Z', 'institution': '${BIRDHOUSE_INSTITUTION}', 'researchSubject': '${BIRDHOUSE_SUBJECT}', 'supportEmail': '${BIRDHOUSE_SUPPORT_EMAIL}', @@ -125,7 +125,7 @@ SERVICES = { 'releasenotes': '${BIRDHOUSE_RELEASE_NOTES_URL}', 'support': '${BIRDHOUSE_SUPPORT_URL}', 'source': 'https://github.com/bird-house/birdhouse-deploy', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}', 'licence': '${BIRDHOUSE_LICENSE_URL}', 'provenance': 'https://pavics-sdi.readthedocs.io/en/latest/provenance/index.html' }, @@ -141,8 +141,8 @@ PLATFORMS = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '2.6.5', - 'releaseTime': '2024-12-18T18:55:30Z', + 'version': '2.7.0', + 'releaseTime': '2024-12-19T13:52:44Z', 'institution': '${BIRDHOUSE_INSTITUTION}', 'researchSubject': '${BIRDHOUSE_SUBJECT}', 'supportEmail': '${BIRDHOUSE_SUPPORT_EMAIL}', @@ -157,7 +157,7 @@ PLATFORMS = { 'releasenotes': 'https://github.com/bird-house/birdhouse-deploy/releases', 'support': 'https://github.com/Ouranosinc/pavics-sdi/issues', 'source': 'https://github.com/Ouranosinc/pavics-sdi', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}', 'licence': 'https://pavics-sdi.readthedocs.io/en/latest/license.html', 'provenance': 'https://pavics-sdi.readthedocs.io/en/latest/provenance/index.html', 'factsheet': 'http://www.canarie.ca/software/pavics' diff --git a/birdhouse/components/cowbird/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/cowbird/config/canarie-api/canarie_api_monitoring.py.template index 1d693d201..55aa4539f 100644 --- a/birdhouse/components/cowbird/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/cowbird/config/canarie-api/canarie_api_monitoring.py.template @@ -19,7 +19,7 @@ SERVICES['Cowbird'] = { 'releasenotes': 'https://github.com/Ouranosinc/cowbird//blob/master/CHANGES.rst', 'support': 'https://github.com/Ouranosinc/cowbird//issues', 'source': 'https://github.com/Ouranosinc/cowbird/', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/cowbird/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/cowbird/', 'licence': 'https://github.com/Ouranosinc/cowbird//blob/${COWBIRD_VERSION}/LICENSE', 'provenance': 'https://github.com/Ouranosinc/cowbird/' }, diff --git a/birdhouse/components/cowbird/config/cowbird/config.yml.template b/birdhouse/components/cowbird/config/cowbird/config.yml.template index fa98db168..3ccbc3b93 100644 --- a/birdhouse/components/cowbird/config/cowbird/config.yml.template +++ b/birdhouse/components/cowbird/config/cowbird/config.yml.template @@ -20,7 +20,7 @@ handlers: admin_password: ${GEOSERVER_ADMIN_PASSWORD} Catalog: active: true - url: https://${BIRDHOUSE_FQDN_PUBLIC}/twitcher/ows/proxy/catalog + url: ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/twitcher/ows/proxy/catalog workspace_dir: ${WORKSPACE_DIR} Thredds: active: true diff --git a/birdhouse/components/cowbird/config/cowbird/cowbird.ini.template b/birdhouse/components/cowbird/config/cowbird/cowbird.ini.template index a1b3daca9..1336f4916 100644 --- a/birdhouse/components/cowbird/config/cowbird/cowbird.ini.template +++ b/birdhouse/components/cowbird/config/cowbird/cowbird.ini.template @@ -32,7 +32,7 @@ mongo_uri = mongodb://${COWBIRD_MONGODB_HOST}:${COWBIRD_MONGODB_PORT}/cowbird # below values are for the external definitions after proxy resolution # internal app access is defined in [server:main] section cowbird.port = -cowbird.url = https://${BIRDHOUSE_FQDN_PUBLIC}/cowbird +cowbird.url = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/cowbird [app:api_app] use = egg:Paste#static diff --git a/birdhouse/components/cowbird/config/proxy/conf.extra-service.d/cowbird.conf.template b/birdhouse/components/cowbird/config/proxy/conf.extra-service.d/cowbird.conf.template index 00d297a06..2c2439001 100644 --- a/birdhouse/components/cowbird/config/proxy/conf.extra-service.d/cowbird.conf.template +++ b/birdhouse/components/cowbird/config/proxy/conf.extra-service.d/cowbird.conf.template @@ -1,6 +1,6 @@ location /cowbird { - proxy_pass https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/cowbird; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/cowbird; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Host $host:$server_port; diff --git a/birdhouse/components/finch/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/finch/config/canarie-api/canarie_api_monitoring.py.template index 58b113166..576f4b78f 100644 --- a/birdhouse/components/finch/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/finch/config/canarie-api/canarie_api_monitoring.py.template @@ -34,7 +34,7 @@ SERVICES['indices'] = { 'monitoring': { 'Finch': { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&version=1.0.0&request=GetCapabilities' } }, } diff --git a/birdhouse/components/finch/service-config.json.template b/birdhouse/components/finch/service-config.json.template index 37642e732..537afa70d 100644 --- a/birdhouse/components/finch/service-config.json.template +++ b/birdhouse/components/finch/service-config.json.template @@ -14,7 +14,7 @@ { "rel": "service", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&request=GetCapabilities" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&request=GetCapabilities" }, { "rel": "service-doc", @@ -24,7 +24,7 @@ { "rel": "service-desc", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&request=GetCapabilities" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&request=GetCapabilities" }, { "rel": "service-meta", diff --git a/birdhouse/components/finch/wps.cfg.template b/birdhouse/components/finch/wps.cfg.template index 19aee28f1..3da46e9bd 100644 --- a/birdhouse/components/finch/wps.cfg.template +++ b/birdhouse/components/finch/wps.cfg.template @@ -1,5 +1,5 @@ [server] -outputurl = https://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs/finch +outputurl = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs/finch outputpath = /data/wpsoutputs/finch # default 3mb, fix "Broken pipe" between the proxy and the wps service diff --git a/birdhouse/components/geoserver/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/geoserver/config/canarie-api/canarie_api_monitoring.py.template index 876128992..1dcd39ad8 100644 --- a/birdhouse/components/geoserver/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/geoserver/config/canarie-api/canarie_api_monitoring.py.template @@ -24,14 +24,14 @@ SERVICES['GeoServer'] = { 'releasenotes': 'https://geoserver.org/release/${GEOSERVER_VERSION}/', 'support': 'https://github.com/kartoza/docker-geoserver/issues', 'source': 'https://github.com/kartoza/docker-geoserver', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/geoserver/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/geoserver/', 'licence': 'https://github.com/geoserver/geoserver/blob/${GEOSERVER_VERSION}/LICENSE.txt', 'provenance': 'https://github.com/kartoza/docker-geoserver' }, "monitoring": { "GeoServer": { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/geoserver/web/' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/geoserver/web/' } } } diff --git a/birdhouse/components/geoserver/config/proxy/conf.extra-service.d/geoserver.conf.template b/birdhouse/components/geoserver/config/proxy/conf.extra-service.d/geoserver.conf.template index 8b3e0de6b..d2b4fd955 100644 --- a/birdhouse/components/geoserver/config/proxy/conf.extra-service.d/geoserver.conf.template +++ b/birdhouse/components/geoserver/config/proxy/conf.extra-service.d/geoserver.conf.template @@ -19,7 +19,7 @@ # If GEOSERVER_SKIP_AUTH is "True" then the following section is skipped and this # location block will always return 200 (which means that the /geoserver/ location, above, # will be publicly available. - proxy_pass https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/geoserver$request_uri; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/geoserver$request_uri; proxy_pass_request_body off; proxy_set_header Host $host; proxy_set_header Content-Length ""; diff --git a/birdhouse/components/geoserver/docker-compose-extra.yml b/birdhouse/components/geoserver/docker-compose-extra.yml index b51c7e395..c78a56a8d 100644 --- a/birdhouse/components/geoserver/docker-compose-extra.yml +++ b/birdhouse/components/geoserver/docker-compose-extra.yml @@ -21,7 +21,7 @@ services: MAXIMUM_MEMORY: 8G # https://github.com/kartoza/docker-geoserver#proxy-base-url HTTP_PROXY_NAME: ${BIRDHOUSE_FQDN_PUBLIC} - HTTP_SCHEME: https + HTTP_SCHEME: ${BIRDHOUSE_PROXY_SCHEME} volumes: # run deployment/fix-geoserver-data-dir-perm on existing # GEOSERVER_DATA_DIR to match user geoserveruser inside docker image diff --git a/birdhouse/components/geoserver/service-config.json.template b/birdhouse/components/geoserver/service-config.json.template index acf7c38aa..bacfb541b 100644 --- a/birdhouse/components/geoserver/service-config.json.template +++ b/birdhouse/components/geoserver/service-config.json.template @@ -20,7 +20,7 @@ { "rel": "service", "type": "text/html", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/geoserver/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/geoserver/" }, { "rel": "service-doc", diff --git a/birdhouse/components/hummingbird/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/hummingbird/config/canarie-api/canarie_api_monitoring.py.template index cfc5ffc83..7d8867b9f 100644 --- a/birdhouse/components/hummingbird/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/hummingbird/config/canarie-api/canarie_api_monitoring.py.template @@ -25,7 +25,7 @@ SERVICES['hummingbird'] = { 'releasenotes': 'https://github.com/bird-house/hummingbird/blob/master/CHANGES.rst', 'support': 'https://github.com/bird-house/hummingbird/issues', 'source': 'https://github.com/bird-house/hummingbird', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird/wps?service=WPS&version=1.0.0&request=GetCapabilities', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird/wps?service=WPS&version=1.0.0&request=GetCapabilities', 'licence': 'https://github.com/bird-house/hummingbird/blob/master/LICENSE.txt', 'provenance': 'https://github.com/bird-house/hummingbird' }, diff --git a/birdhouse/components/hummingbird/service-config.json.template b/birdhouse/components/hummingbird/service-config.json.template index 7e36d64c2..4cc5e453a 100644 --- a/birdhouse/components/hummingbird/service-config.json.template +++ b/birdhouse/components/hummingbird/service-config.json.template @@ -14,7 +14,7 @@ { "rel": "service", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&request=GetCapabilities" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&request=GetCapabilities" }, { "rel": "service-doc", @@ -24,7 +24,7 @@ { "rel": "service-desc", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&request=GetCapabilities" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&request=GetCapabilities" }, { "rel": "service-meta", diff --git a/birdhouse/components/jupyterhub/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/jupyterhub/config/canarie-api/canarie_api_monitoring.py.template index 1681c8804..ba9abdc56 100644 --- a/birdhouse/components/jupyterhub/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/jupyterhub/config/canarie-api/canarie_api_monitoring.py.template @@ -19,14 +19,14 @@ SERVICES['Jupyter'] = { 'releasenotes': 'https://github.com/Ouranosinc/jupyterhub/tags', # no CHANGES file available 'support': 'https://github.com/Ouranosinc/jupyterhub/issues', 'source': 'https://github.com/Ouranosinc/jupyterhub', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/jupyter/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/jupyter/', 'licence': 'https://github.com/Ouranosinc/jupyterhub/blob/${JUPYTERHUB_VERSION}/LICENSE', 'provenance': '' }, "monitoring": { "Jupyter": { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/jupyter/hub/login' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/jupyter/hub/login' }, } } diff --git a/birdhouse/components/jupyterhub/docker-compose-extra.yml b/birdhouse/components/jupyterhub/docker-compose-extra.yml index 5c2cc0a1d..2ee32650a 100644 --- a/birdhouse/components/jupyterhub/docker-compose-extra.yml +++ b/birdhouse/components/jupyterhub/docker-compose-extra.yml @@ -28,6 +28,7 @@ services: USER_WORKSPACE_UID: ${USER_WORKSPACE_UID} USER_WORKSPACE_GID: ${USER_WORKSPACE_GID} JUPYTERHUB_CRYPT_KEY: ${JUPYTERHUB_CRYPT_KEY} + JUPYTERHUB_DOCKER_EXTRA_HOSTS: ${JUPYTERHUB_DOCKER_EXTRA_HOSTS:-} volumes: - ./components/jupyterhub/jupyterhub_config.py:/srv/jupyterhub/jupyterhub_config.py:ro - ./components/jupyterhub/custom_templates:/custom_templates:ro diff --git a/birdhouse/components/jupyterhub/jupyterhub_config.py.template b/birdhouse/components/jupyterhub/jupyterhub_config.py.template index 0fce675b0..ad93e5058 100644 --- a/birdhouse/components/jupyterhub/jupyterhub_config.py.template +++ b/birdhouse/components/jupyterhub/jupyterhub_config.py.template @@ -107,10 +107,10 @@ c.DockerSpawner.environment = { # Post on Panel forum: # https://discourse.holoviz.org/t/how-to-customize-the-display-url-from-panel-serve-for-use-behind-jupyterhub-with-jupyter-server-proxy/3571 # Issue about Panel Preview: https://github.com/holoviz/panel/issues/3440 - "BIRDHOUSE_HOST_URL": "https://${BIRDHOUSE_FQDN_PUBLIC}", + "BIRDHOUSE_HOST_URL": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}", # https://docs.dask.org/en/stable/configuration.html # https://jupyterhub-on-hadoop.readthedocs.io/en/latest/dask.html - "DASK_DISTRIBUTED__DASHBOARD__LINK": "https://${BIRDHOUSE_FQDN_PUBLIC}{JUPYTERHUB_SERVICE_PREFIX}proxy/{port}/status" + "DASK_DISTRIBUTED__DASHBOARD__LINK": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}{JUPYTERHUB_SERVICE_PREFIX}proxy/{port}/status" } host_user_data_dir = join(os.environ['WORKSPACE_DIR'], "{username}") @@ -200,9 +200,11 @@ c.Spawner.args = [ "--FileContentsManager.always_delete_dir=True", ] +## Note that JUPYTERHUB_DOCKER_EXTRA_HOSTS may be set by default in the local-dev-test component c.DockerSpawner.extra_host_config = { # start init pid 1 process to reap defunct processes 'init': True, + 'extra_hosts': dict(host_mapping.split(":") for host_mapping in os.getenv("JUPYTERHUB_DOCKER_EXTRA_HOSTS", "").split()) } c.Authenticator.admin_users = ${JUPYTERHUB_ADMIN_USERS} # noqa diff --git a/birdhouse/components/jupyterhub/service-config.json.template b/birdhouse/components/jupyterhub/service-config.json.template index 59850d054..0d6fcfff2 100644 --- a/birdhouse/components/jupyterhub/service-config.json.template +++ b/birdhouse/components/jupyterhub/service-config.json.template @@ -14,7 +14,7 @@ { "rel": "service", "type": "text/html", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/jupyter" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/jupyter" }, { "rel": "service-doc", diff --git a/birdhouse/components/magpie/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/magpie/config/canarie-api/canarie_api_monitoring.py.template index 0f9e4b23f..b4eee3c44 100644 --- a/birdhouse/components/magpie/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/magpie/config/canarie-api/canarie_api_monitoring.py.template @@ -23,14 +23,14 @@ SERVICES['Magpie'] = { 'releasenotes': 'https://github.com/Ouranosinc/Magpie/blob/master/CHANGES.rst', 'support': 'https://github.com/Ouranosinc/Magpie/issues', 'source': 'https://github.com/Ouranosinc/Magpie', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/magpie/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/magpie/', 'licence': 'https://github.com/Ouranosinc/Magpie/blob/${MAGPIE_VERSION}/LICENSE', 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' }, "monitoring": { "Magpie": { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/magpie/version' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/magpie/version' }, 'response': { 'text': r'\{.*"code": 200.*"type": "application/json".*\}' diff --git a/birdhouse/components/magpie/docker-compose-extra.yml b/birdhouse/components/magpie/docker-compose-extra.yml index 4c464fd3c..737cd080a 100644 --- a/birdhouse/components/magpie/docker-compose-extra.yml +++ b/birdhouse/components/magpie/docker-compose-extra.yml @@ -12,7 +12,7 @@ services: image: pavics/magpie:${MAGPIE_VERSION} container_name: magpie environment: - TWITCHER_PROTECTED_URL: https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH} + TWITCHER_PROTECTED_URL: ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH} # target directories to allow loading multiple config files of corresponding category # each compose override should volume mount its files inside the matching directories # (note: DO NOT use 'MAGPIE_CONFIG_PATH' that would disable multi-config loading capability) diff --git a/birdhouse/components/magpie/magpie.ini.template b/birdhouse/components/magpie/magpie.ini.template index f1c0ed93d..58e6b84da 100644 --- a/birdhouse/components/magpie/magpie.ini.template +++ b/birdhouse/components/magpie/magpie.ini.template @@ -28,7 +28,7 @@ pyramid.includes = pyramid_tm ziggurat_foundations.ext.pyramid.sign_in ziggurat_ # other overridable variables available in magpie/constants.py # magpie.port = 2001 -magpie.url = https://${BIRDHOUSE_FQDN}/magpie +magpie.url = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/magpie magpie.max_restart = 5 magpie.push_phoenix = true # This secret should be the same in Twitcher ! diff --git a/birdhouse/components/monitoring/config/proxy/conf.extra-service.d/monitoring.conf.template b/birdhouse/components/monitoring/config/proxy/conf.extra-service.d/monitoring.conf.template index 07a62c224..95ac63d8b 100644 --- a/birdhouse/components/monitoring/config/proxy/conf.extra-service.d/monitoring.conf.template +++ b/birdhouse/components/monitoring/config/proxy/conf.extra-service.d/monitoring.conf.template @@ -22,7 +22,7 @@ location = /secure-grafana-auth { internal; - proxy_pass https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/grafana$request_uri; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/grafana$request_uri; proxy_pass_request_body off; proxy_set_header Host $host; proxy_set_header Content-Length ""; @@ -34,7 +34,7 @@ location = /secure-prometheus-auth { internal; - proxy_pass https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/prometheus$request_uri; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/prometheus$request_uri; proxy_pass_request_body off; proxy_set_header Host $host; proxy_set_header Content-Length ""; @@ -46,7 +46,7 @@ location = /secure-alertmanager-auth { internal; - proxy_pass https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/alertmanager$request_uri; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/alertmanager$request_uri; proxy_pass_request_body off; proxy_set_header Host $host; proxy_set_header Content-Length ""; diff --git a/birdhouse/components/monitoring/default.env b/birdhouse/components/monitoring/default.env index 76188073d..5c9f3158b 100644 --- a/birdhouse/components/monitoring/default.env +++ b/birdhouse/components/monitoring/default.env @@ -19,6 +19,7 @@ export CADVISOR_IMAGE='${CADVISOR_DOCKER}:${CADVISOR_VERSION}' export NODE_EXPORTER_VERSION="v1.0.0" export NODE_EXPORTER_DOCKER="quay.io/prometheus/node-exporter" export NODE_EXPORTER_IMAGE='${NODE_EXPORTER_DOCKER}:${NODE_EXPORTER_VERSION}' +export NODE_EXPORTER_HOST_BIND_PROPOGATION=rslave export ALERTMANAGER_VERSION="v0.21.0" export ALERTMANAGER_DOCKER=prom/alertmanager diff --git a/birdhouse/components/monitoring/docker-compose-extra.yml b/birdhouse/components/monitoring/docker-compose-extra.yml index 29f6f46e9..261826c83 100644 --- a/birdhouse/components/monitoring/docker-compose-extra.yml +++ b/birdhouse/components/monitoring/docker-compose-extra.yml @@ -21,7 +21,7 @@ services: image: ${NODE_EXPORTER_IMAGE} container_name: node-exporter volumes: - - /:/host:ro,rslave + - /:/host:ro,${NODE_EXPORTER_HOST_BIND_PROPOGATION} network_mode: "host" pid: "host" command: --path.rootfs=/host @@ -45,7 +45,7 @@ services: # https://prometheus.io/docs/prometheus/latest/storage/ - --storage.tsdb.retention.time=90d # wrong default was http://container-hash:9090/ - - --web.external-url=https://${BIRDHOUSE_FQDN_PUBLIC}/prometheus/ + - --web.external-url=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/prometheus/ restart: always # https://grafana.com/docs/grafana/latest/installation/docker/ @@ -61,7 +61,7 @@ services: - grafana_persistence:/var/lib/grafana:rw environment: GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} - GF_SERVER_ROOT_URL: https://${BIRDHOUSE_FQDN_PUBLIC}/grafana + GF_SERVER_ROOT_URL: ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/grafana GF_SERVER_SERVE_FROM_SUB_PATH: 'true' GF_SERVER_DOMAIN: ${BIRDHOUSE_FQDN_PUBLIC} restart: always @@ -83,7 +83,7 @@ services: # enable debug logging - --log.level=debug # wrong default was http://container-hash:9093/ - - --web.external-url=https://${BIRDHOUSE_FQDN_PUBLIC}/alertmanager + - --web.external-url=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/alertmanager restart: always volumes: diff --git a/birdhouse/components/proxy/conf.d/frontend.conf.template b/birdhouse/components/proxy/conf.d/frontend.conf.template index 391f7902c..4eb872aae 100644 --- a/birdhouse/components/proxy/conf.d/frontend.conf.template +++ b/birdhouse/components/proxy/conf.d/frontend.conf.template @@ -18,7 +18,7 @@ server { listen 80; server_name localhost; - ${INCLUDE_FOR_PORT_80} + ${PROXY_INCLUDE_FOR_PORT_80} # redirect server error pages to the static page /50x.html error_page 500 502 503 504 /50x.html; @@ -27,22 +27,4 @@ server { } } -server { - listen 443; - server_name localhost; - proxy_buffering off; - - resolver 127.0.0.11; - - ssl on; - ssl_certificate /etc/nginx/cert.pem; - ssl_certificate_key /etc/nginx/cert.pem; - - include /etc/nginx/conf.d/all-services.include; - - # redirect server error pages to the static page /50x.html - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } -} +${PROXY_INCLUDE_HTTPS} diff --git a/birdhouse/components/proxy/conf.d/https.include b/birdhouse/components/proxy/conf.d/https.include new file mode 100644 index 000000000..992fd2458 --- /dev/null +++ b/birdhouse/components/proxy/conf.d/https.include @@ -0,0 +1,19 @@ +server { + listen 443; + server_name localhost; + proxy_buffering off; + + resolver 127.0.0.11; + + ssl on; + ssl_certificate /etc/nginx/cert.pem; + ssl_certificate_key /etc/nginx/cert.pem; + + include /etc/nginx/conf.d/all-services.include; + + # redirect server error pages to the static page /50x.html + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} \ No newline at end of file diff --git a/birdhouse/components/proxy/default.env b/birdhouse/components/proxy/default.env index 08f9dde89..1b28892ec 100644 --- a/birdhouse/components/proxy/default.env +++ b/birdhouse/components/proxy/default.env @@ -6,14 +6,18 @@ export PROXY_IMAGE="nginx:1.23.4" # Any WPS processes taking longer than this should use async mode. export PROXY_READ_TIMEOUT_VALUE="240s" +export BIRDHOUSE_PROXY_SCHEME='$([ x"$BIRDHOUSE_HTTP_ONLY" = x"True" ] && echo http || echo https )' +export BIRDHOUSE_ALLOW_UNSECURE_HTTP='$([ x"$BIRDHOUSE_HTTP_ONLY" = x"True" ] && echo True || echo )' +export PROXY_INCLUDE_HTTPS='$([ x"$BIRDHOUSE_HTTP_ONLY" = x"True" ] || echo "include /etc/nginx/conf.d/https.include;" )' + # Content of "location /" in file config/proxy/conf.d/all-services.include.template # Useful to have a custom homepage. # Note that the default homepage will become the jupyterhub login page if the jupyterhub component is enabled. # If the jupyterhub component is not enabled, it is highly recommended to create a custom homepage since the magpie # landing page is not the most user-friendly option. -export BIRDHOUSE_PROXY_ROOT_LOCATION="return 302 https://\$host/jupyter/hub/login;" +export BIRDHOUSE_PROXY_ROOT_LOCATION='return 302 ${BIRDHOUSE_PROXY_SCHEME}://\$host/jupyter/hub/login;' -export INCLUDE_FOR_PORT_80='$([ x"$BIRDHOUSE_ALLOW_UNSECURE_HTTP" = x"True" ] && echo "include /etc/nginx/conf.d/all-services.include;" || echo "include /etc/nginx/conf.d/redirect-to-https.include;")' +export PROXY_INCLUDE_FOR_PORT_80='$([ x"$BIRDHOUSE_ALLOW_UNSECURE_HTTP" = x"True" ] && echo "include /etc/nginx/conf.d/all-services.include;" || echo "include /etc/nginx/conf.d/redirect-to-https.include;")' export PROXY_LOG_DIR="/var/log/nginx/" export PROXY_LOG_FILE="access_file.log" @@ -21,8 +25,12 @@ export PROXY_LOG_PATH='${PROXY_LOG_DIR}/${PROXY_LOG_FILE}' export DELAYED_EVAL=" $DELAYED_EVAL - INCLUDE_FOR_PORT_80 PROXY_LOG_PATH + BIRDHOUSE_PROXY_SCHEME + BIRDHOUSE_ALLOW_UNSECURE_HTTP + PROXY_INCLUDE_HTTPS + PROXY_INCLUDE_FOR_PORT_80 + BIRDHOUSE_PROXY_ROOT_LOCATION " # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here @@ -31,13 +39,15 @@ export VARS=" \$BIRDHOUSE_DEPLOY_COMPONENTS_JSON \$BIRDHOUSE_DEPLOY_SERVICES_JSON \$BIRDHOUSE_VERSION_JSON + \$BIRDHOUSE_PROXY_SCHEME " export OPTIONAL_VARS=" $OPTIONAL_VARS - \$INCLUDE_FOR_PORT_80 + \$PROXY_INCLUDE_FOR_PORT_80 \$PROXY_READ_TIMEOUT_VALUE \$BIRDHOUSE_PROXY_ROOT_LOCATION \$PROXY_LOG_FILE \$PROXY_LOG_PATH + \$PROXY_INCLUDE_HTTPS " diff --git a/birdhouse/components/proxy/docker-compose-extra.yml b/birdhouse/components/proxy/docker-compose-extra.yml index 66ce09ef4..86852670b 100644 --- a/birdhouse/components/proxy/docker-compose-extra.yml +++ b/birdhouse/components/proxy/docker-compose-extra.yml @@ -12,12 +12,10 @@ services: image: ${PROXY_IMAGE} container_name: proxy ports: - - "80:80" - - "443:${PROXY_SECURE_PORT}" + - "80:${PROXY_HTTP_PORT}" volumes: - ./components/proxy/conf.d:/etc/nginx/conf.d - ./components/proxy/nginx.conf:/etc/nginx/nginx.conf - - ${BIRDHOUSE_SSL_CERTIFICATE}:/etc/nginx/cert.pem - ./components/proxy/static:/static environment: # https://github.com/bird-house/birdhouse-deploy/issues/198 diff --git a/birdhouse/components/proxy/docker-compose-ssl-cert.yml b/birdhouse/components/proxy/docker-compose-ssl-cert.yml new file mode 100644 index 000000000..abd2a4a95 --- /dev/null +++ b/birdhouse/components/proxy/docker-compose-ssl-cert.yml @@ -0,0 +1,8 @@ +version: "3.4" + +services: + proxy: + ports: + - "443:443" + volumes: + - ${BIRDHOUSE_SSL_CERTIFICATE}:/etc/nginx/cert.pem diff --git a/birdhouse/components/proxy/pre-docker-compose-up.include b/birdhouse/components/proxy/pre-docker-compose-up.include new file mode 100644 index 000000000..b46695038 --- /dev/null +++ b/birdhouse/components/proxy/pre-docker-compose-up.include @@ -0,0 +1,13 @@ +#!/usr/bin/env sh + +# Note: filename is not docker-compose-extra.yml so that it won't get added prematurely to COMPOSE_CONF_LIST + +if [ x"$BIRDHOUSE_HTTP_ONLY" != x"True" ]; then + THIS_COMPOSE_FILE="${COMPOSE_DIR}/components/proxy/docker-compose-ssl-cert.yml" + + COMPOSE_CONF_LIST="${COMPOSE_CONF_LIST} -f ${THIS_COMPOSE_FILE}" + + log INFO "adding ${THIS_COMPOSE_FILE} to COMPOSE_CONF_LIST" + + unset THIS_COMPOSE_FILE +fi diff --git a/birdhouse/components/raven/default.env b/birdhouse/components/raven/default.env index 24f115a56..43c4047f0 100644 --- a/birdhouse/components/raven/default.env +++ b/birdhouse/components/raven/default.env @@ -4,7 +4,7 @@ # This is the production Geoserver that is always available with appropriate data. # For site that want to run your own Geoserver with your own data, please # override this variable with your own Geoserver instance. -# Ex: RAVEN_GEO_URL="https://${BIRDHOUSE_FQDN}/geoserver/" +# Ex: RAVEN_GEO_URL="${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/geoserver/" __DEFAULT__RAVEN_GEO_URL="https://pavics.ouranos.ca/geoserver/" export RAVEN_GEO_URL='${__DEFAULT__RAVEN_GEO_URL}' diff --git a/birdhouse/components/raven/service-config.json.template b/birdhouse/components/raven/service-config.json.template index 96517aea4..40e00b616 100644 --- a/birdhouse/components/raven/service-config.json.template +++ b/birdhouse/components/raven/service-config.json.template @@ -14,7 +14,7 @@ { "rel": "service", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&request=GetCapabilities" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&request=GetCapabilities" }, { "rel": "service-doc", @@ -24,7 +24,7 @@ { "rel": "service-desc", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&request=GetCapabilities" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&request=GetCapabilities" }, { "rel": "service-meta", diff --git a/birdhouse/components/raven/wps.cfg.template b/birdhouse/components/raven/wps.cfg.template index 822129ee5..4596a5c09 100644 --- a/birdhouse/components/raven/wps.cfg.template +++ b/birdhouse/components/raven/wps.cfg.template @@ -1,5 +1,5 @@ [server] -outputurl = https://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs/raven +outputurl = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs/raven outputpath = /data/wpsoutputs/raven # default 3mb, fix "Broken pipe" between the proxy and the wps service diff --git a/birdhouse/components/stac/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/stac/config/canarie-api/canarie_api_monitoring.py.template index 360fd0387..d3e280cae 100644 --- a/birdhouse/components/stac/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/stac/config/canarie-api/canarie_api_monitoring.py.template @@ -19,7 +19,7 @@ SERVICES['STAC'] = { 'releasenotes': 'https://github.com/crim-ca/sac-app/blob/master/CHANGES.rst', 'support': 'https://github.com/crim-ca/stac-app/issues', 'source': 'https://github.com/crim-ca/stac-app', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/stac/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/stac/', 'licence': 'https://github.com/crim-ca/stac-app/blob/master/LICENSE', 'provenance': 'https://github.com/crim-ca/stac-app' }, diff --git a/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template b/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template index cdaee3575..8db59fcc5 100644 --- a/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template +++ b/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template @@ -4,7 +4,7 @@ # We need the second `/stac` for API redirect in STAC (see `root-path` and `ROUTER_PREFIX`). # See https://github.com/stac-utils/stac-fastapi/issues/427 # See https://github.com/crim-ca/stac-app/blob/main/stac_app.py#L60 - proxy_pass https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/stac/stac; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/stac/stac/; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Host $host:$server_port; @@ -14,7 +14,7 @@ # Automatically redirect to /stac/stac and exclude redirect when already using /stac location ~ ^${TWITCHER_PROTECTED_PATH}/stac(?!/stac) { - return 302 ${TWITCHER_PROTECTED_PATH}/stac/stac; + return 302 ${TWITCHER_PROTECTED_PATH}/stac/stac/; } location /stac-browser/ { diff --git a/birdhouse/components/stac/docker-compose-extra.yml b/birdhouse/components/stac/docker-compose-extra.yml index 2362aa967..7e62a290c 100644 --- a/birdhouse/components/stac/docker-compose-extra.yml +++ b/birdhouse/components/stac/docker-compose-extra.yml @@ -30,7 +30,7 @@ services: container_name: stac-browser image: ${STAC_BROWSER_IMAGE} environment: - - CATALOG_URL=https://${BIRDHOUSE_FQDN_PUBLIC}/stac/ + - CATALOG_URL=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/stac/ - ROOT_PATH=/stac-browser/ stac-db: diff --git a/birdhouse/components/stac/service-config.json.template b/birdhouse/components/stac/service-config.json.template index 5334fbc36..7508e3b22 100644 --- a/birdhouse/components/stac/service-config.json.template +++ b/birdhouse/components/stac/service-config.json.template @@ -15,7 +15,7 @@ { "rel": "service", "type": "application/json", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/stac/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/stac/" }, { "rel": "service-doc", @@ -30,7 +30,7 @@ { "rel": "alternate", "type": "text/html", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/stac-browser/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/stac-browser/" }, { "rel": "service-meta", @@ -55,7 +55,7 @@ { "rel": "service", "type": "text/html", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/stac-browser/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/stac-browser/" }, { "rel": "service-doc", @@ -65,7 +65,7 @@ { "rel": "alternate", "type": "application/json", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/stac/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/stac/" }, { "rel": "service-meta", diff --git a/birdhouse/components/thredds/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/thredds/config/canarie-api/canarie_api_monitoring.py.template index 904723802..0e6db19a7 100644 --- a/birdhouse/components/thredds/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/thredds/config/canarie-api/canarie_api_monitoring.py.template @@ -29,7 +29,7 @@ SERVICES['renderer'] = { 'monitoring': { 'ncWMS': { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/wms/${THREDDS_SERVICE_DATA_URL_PATH}/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/wms/${THREDDS_SERVICE_DATA_URL_PATH}/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' } }, } @@ -56,7 +56,7 @@ SERVICES['Thredds'] = { 'releasenotes': 'https://docs.unidata.ucar.edu/tds/current/userguide/upgrade.html', 'support': 'https://www.unidata.ucar.edu/software/tds/#help', 'source': 'https://github.com/Unidata/tds', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/', 'licence': 'https://github.com/Unidata/tds/blob/main/LICENSE', 'provenance': 'https://downloads.unidata.ucar.edu/tds/' }, diff --git a/birdhouse/components/thredds/config/proxy/conf.extra-service.d/thredds.conf.template b/birdhouse/components/thredds/config/proxy/conf.extra-service.d/thredds.conf.template index 671da15ab..b78c0276f 100644 --- a/birdhouse/components/thredds/config/proxy/conf.extra-service.d/thredds.conf.template +++ b/birdhouse/components/thredds/config/proxy/conf.extra-service.d/thredds.conf.template @@ -1,7 +1,7 @@ location /thredds/ { #return 302 /twitcher/ows/proxy$request_uri; - proxy_pass https://${BIRDHOUSE_FQDN}${TWITCHER_PROTECTED_PATH}/thredds/; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}${TWITCHER_PROTECTED_PATH}/thredds/; # direct hit Thredds, bypassing twitcher, for debugging only # proxy_pass http://thredds:8080${TWITCHER_PROTECTED_PATH}/thredds/; proxy_set_header Host $host; diff --git a/birdhouse/components/thredds/service-config.json.template b/birdhouse/components/thredds/service-config.json.template index 9fe03f9f6..004fe099a 100644 --- a/birdhouse/components/thredds/service-config.json.template +++ b/birdhouse/components/thredds/service-config.json.template @@ -16,7 +16,7 @@ { "rel": "service", "type": "text/html", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/thredds/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/thredds/" }, { "rel": "service-doc", @@ -26,7 +26,7 @@ { "rel": "service-desc", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/thredds/catalog.xml" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/thredds/catalog.xml" }, { "rel": "service-meta", diff --git a/birdhouse/components/twitcher/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/twitcher/config/canarie-api/canarie_api_monitoring.py.template index b5576e122..f7addaa7a 100644 --- a/birdhouse/components/twitcher/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/twitcher/config/canarie-api/canarie_api_monitoring.py.template @@ -42,14 +42,14 @@ SERVICES['Twitcher'] = { 'releasenotes': 'https://github.com/bird-house/twitcher/blob/master/CHANGES.rst', 'support': 'https://github.com/bird-house/twitcher/issues', 'source': 'https://github.com/bird-house/twitcher', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/twitcher/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/twitcher/', 'licence': 'https://github.com/bird-house/twitcher/blob/master/LICENSE.txt', 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' }, "monitoring": { "Twitcher": { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/twitcher/' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/twitcher/' } } } diff --git a/birdhouse/components/twitcher/config/proxy/conf.extra-service.d/twitcher.conf.template b/birdhouse/components/twitcher/config/proxy/conf.extra-service.d/twitcher.conf.template index 2a7e897eb..44b558eaf 100644 --- a/birdhouse/components/twitcher/config/proxy/conf.extra-service.d/twitcher.conf.template +++ b/birdhouse/components/twitcher/config/proxy/conf.extra-service.d/twitcher.conf.template @@ -4,5 +4,5 @@ proxy_set_header X-Forwarded-Proto $real_scheme; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Host $host:$server_port; - proxy_set_header Forwarded "proto=https;host=${BIRDHOUSE_FQDN_PUBLIC}"; # Helps the STAC component to craft URLs containing the full BIRDHOUSE_FQDN_PUBLIC + proxy_set_header Forwarded "proto=${BIRDHOUSE_PROXY_SCHEME};host=${BIRDHOUSE_FQDN_PUBLIC}"; # Helps the STAC component to craft URLs containing the full BIRDHOUSE_FQDN_PUBLIC } diff --git a/birdhouse/components/twitcher/twitcher.ini.template b/birdhouse/components/twitcher/twitcher.ini.template index e00479012..02da6b4cb 100644 --- a/birdhouse/components/twitcher/twitcher.ini.template +++ b/birdhouse/components/twitcher/twitcher.ini.template @@ -51,7 +51,7 @@ cache.service.expire = 60 # debugtoolbar.hosts = 127.0.0.1 ::1 # twitcher -twitcher.url = https://${BIRDHOUSE_FQDN}/twitcher +twitcher.url = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/twitcher twitcher.adapter = magpie.adapter.MagpieAdapter twitcher.rpcinterface = false twitcher.username = @@ -65,7 +65,7 @@ twitcher.workdir = twitcher.prefix = # magpie (for twitcher.adapter) -magpie.url = https://${BIRDHOUSE_FQDN}/magpie +magpie.url = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/magpie magpie.secret = ${MAGPIE_SECRET} magpie.admin_user = ${MAGPIE_ADMIN_USERNAME} magpie.admin_password = ${MAGPIE_ADMIN_PASSWORD} diff --git a/birdhouse/components/weaver/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/weaver/config/canarie-api/canarie_api_monitoring.py.template index 21f8a8a35..6ba1c83a2 100644 --- a/birdhouse/components/weaver/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/weaver/config/canarie-api/canarie_api_monitoring.py.template @@ -19,7 +19,7 @@ SERVICES['Weaver'] = { 'releasenotes': 'https://github.com/crim-ca/weaver/blob/master/CHANGES.rst', 'support': 'https://github.com/crim-ca/weaver/issues', 'source': 'https://github.com/crim-ca/weaver', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/weaver/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/', 'licence': 'https://github.com/crim-ca/weaver/blob/${WEAVER_VERSION}/LICENSE.txt', 'provenance': 'https://github.com/crim-ca/weaver' }, diff --git a/birdhouse/components/weaver/config/proxy/conf.extra-service.d/weaver.conf.template b/birdhouse/components/weaver/config/proxy/conf.extra-service.d/weaver.conf.template index 6a0d99820..9a3d5b91e 100644 --- a/birdhouse/components/weaver/config/proxy/conf.extra-service.d/weaver.conf.template +++ b/birdhouse/components/weaver/config/proxy/conf.extra-service.d/weaver.conf.template @@ -6,11 +6,11 @@ # whether the *shortcut* Weaver endpoint, the alias or the explicit 'twitcher' proxy route is used. # redirect EMS/ADES to actual secured Weaver path #location /${WEAVER_CONFIG} { - # return 302 https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}; + # return 302 ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}; #} location /${WEAVER_MANAGER_NAME} { - proxy_pass https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}; proxy_set_header Host $host; proxy_buffering off; include /etc/nginx/conf.d/cors.include; diff --git a/birdhouse/components/weaver/config/weaver/data_sources.yml.template b/birdhouse/components/weaver/config/weaver/data_sources.yml.template index 7ad9dbb13..6e8e8c520 100644 --- a/birdhouse/components/weaver/config/weaver/data_sources.yml.template +++ b/birdhouse/components/weaver/config/weaver/data_sources.yml.template @@ -2,13 +2,13 @@ # Employed by default for looking at files available directly (e.g.: pre-fetched) localhost: netloc: "localhost" - ades: "https://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}" + ades: "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}" default: true # Weaver self-reference, but using the exposed endpoint on the birdhouse instance public: netloc: "${BIRDHOUSE_FQDN_PUBLIC}" - ades: "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}" + ades: "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}" # Weaver self-reference with specifically 'opensearch' parameters as process input. # Since none are available in the stack, there is no collection ID. diff --git a/birdhouse/components/weaver/config/weaver/weaver.ini.template b/birdhouse/components/weaver/config/weaver/weaver.ini.template index 19e3b2647..91236355d 100644 --- a/birdhouse/components/weaver/config/weaver/weaver.ini.template +++ b/birdhouse/components/weaver/config/weaver/weaver.ini.template @@ -24,7 +24,7 @@ mongodb.db_name = weaver # --- Weaver Configuration --- weaver.configuration = ${WEAVER_CONFIG} -weaver.url = https://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME} +weaver.url = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME} # --- Weaver requests extension flags --- # use request-options for per-request specific disabling instead of globally disabled ssl_verify flag @@ -49,7 +49,7 @@ weaver.wps = true weaver.wps_path = ${WEAVER_WPS_PATH} weaver.wps_output = true weaver.wps_output_dir = ${WEAVER_WPS_OUTPUTS_DIR} -weaver.wps_output_url = https://${BIRDHOUSE_FQDN_PUBLIC}${WEAVER_WPS_OUTPUTS_PATH} +weaver.wps_output_url = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${WEAVER_WPS_OUTPUTS_PATH} weaver.wps_output_path = # default output sub-dir if user not specified (just for good measure, otherwise hook should default to it also) # see 'components/weaver/config/magpie/weaver_hooks.py:add_x_wps_output_context' that defines it when user is logged in @@ -60,7 +60,7 @@ weaver.wps_workdir = ${WEAVER_WPS_WORKDIR} weaver.wps_metadata_identification_title=Weaver weaver.wps_metadata_identification_abstract=Weaver is an Execution Management Service (EMS) that allows the execution of workflows chaining various applications and Web Processing Services (WPS) inputs and outputs. Remote execution is deferred by the EMS to an Application Deployment and Execution Service (ADES), as defined by Common Workflow Language (CWL) configurations. weaver.wps_metadata_identification_keywords=Weaver,WPS,OGC,${WEAVER_CONFIG} -weaver.wps_metadata_identification_accessconstraints=https://${BIRDHOUSE_FQDN_PUBLIC}/magpie +weaver.wps_metadata_identification_accessconstraints=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/magpie weaver.wps_metadata_identification_fees=NONE weaver.wps_metadata_provider_name=crim-ca/weaver weaver.wps_metadata_provider_url=http://pavics-weaver.readthedocs.org/en/latest/ diff --git a/birdhouse/components/weaver/post-docker-compose-up b/birdhouse/components/weaver/post-docker-compose-up index 5cb09b3e9..d4dbcfbc2 100755 --- a/birdhouse/components/weaver/post-docker-compose-up +++ b/birdhouse/components/weaver/post-docker-compose-up @@ -23,7 +23,7 @@ # # WEAVER_WPS_PROVIDERS: # list of provider names (comma or space delimited), all are assumed to be available at -# "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/" +# "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/" # WEAVER_WPS_PROVIDERS_MAX_TIME: # limit script execution up to a maximum of this number of seconds # WEAVER_WPS_PROVIDERS_RETRY_COUNT: @@ -82,8 +82,8 @@ WARN="${PREFIX}${YELLOW}WARNING${NORMAL}: " echo "${PREFIX}Running: $0" -MAGPIE_URL="https://${BIRDHOUSE_FQDN_PUBLIC}/magpie" -WEAVER_URL="https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}" +MAGPIE_URL="${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/magpie" +WEAVER_URL="${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}" WEAVER_WPS_PROVIDERS_MAX_TIME=${WEAVER_WPS_PROVIDERS_MAX_TIME:-120} WEAVER_WPS_PROVIDERS_RETRY_AFTER=${WEAVER_WPS_PROVIDERS_RETRY_AFTER:-5} WEAVER_WPS_PROVIDERS_RETRY_COUNT=${WEAVER_WPS_PROVIDERS_RETRY_COUNT:-5} @@ -246,7 +246,7 @@ for prov in ${WEAVER_WPS_PROVIDERS}; do if [ -z "${prov}" ]; then continue fi - prov_url="https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${prov}" + prov_url="${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${prov}" prov_cap="${prov_url}?service=WPS&request=GetCapabilities" # wait for WPS provider to respond diff --git a/birdhouse/components/weaver/service-config.json.template b/birdhouse/components/weaver/service-config.json.template index 0e3f79fcb..a9acfd807 100644 --- a/birdhouse/components/weaver/service-config.json.template +++ b/birdhouse/components/weaver/service-config.json.template @@ -15,7 +15,7 @@ { "rel": "service", "type": "application/json", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/" }, { "rel": "service-doc", @@ -25,12 +25,12 @@ { "rel": "service-desc", "type": "application/json", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/" }, { "rel": "conformance", "type": "application/json", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/conformance/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/conformance/" }, { "rel": "service-meta", diff --git a/birdhouse/deprecated-components/catalog/catalog.cfg.template b/birdhouse/deprecated-components/catalog/catalog.cfg.template index 07d6a71f0..a7ab2c7e5 100644 --- a/birdhouse/deprecated-components/catalog/catalog.cfg.template +++ b/birdhouse/deprecated-components/catalog/catalog.cfg.template @@ -3,7 +3,7 @@ solr_host=http://${BIRDHOUSE_FQDN}:8983/solr/${THREDDS_SERVICE_DATA_URL_PATH}/ # Multiple thredds hosts can be given, comma separated # note: this URL is also used as prefix when comparing authorizations from magpie -thredds_host=https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds +thredds_host=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds # Multiple esgf nodes can be given, comma separated esgf_nodes=https://esgf-node.llnl.gov/esg-search @@ -11,7 +11,7 @@ esgf_nodes=https://esgf-node.llnl.gov/esg-search # Provide a magpie host to filter results based on access permissions. # Must also provide credentials with read access so that the crawler can parse the thredds host(s) # Leave as a comment for a public catalog. -magpie_host=https://${BIRDHOUSE_FQDN_PUBLIC}/magpie +magpie_host=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/magpie magpie_user=${CATALOG_USERNAME} magpie_pw=${CATALOG_PASSWORD} # SSL verification (true or false) @@ -22,10 +22,10 @@ thredds_host_magpie_svc_name=${CATALOG_THREDDS_SERVICE} # WMS service url with replaced by each instance of the thredds_host, # without the port number and replaced by the base url in thredds. # Leave as comment to use the default WMS service -wms_alternate_server=https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/ncWMS2/wms?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0&DATASET=outputs/ +wms_alternate_server=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/ncWMS2/wms?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0&DATASET=outputs/ [pywps] -outputurl=https://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs/catalog +outputurl=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs/catalog parallelprocesses=30 [logging] diff --git a/birdhouse/deprecated-components/flyingpigeon/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/deprecated-components/flyingpigeon/config/canarie-api/canarie_api_monitoring.py.template index ec4110010..1dd847a0b 100644 --- a/birdhouse/deprecated-components/flyingpigeon/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/deprecated-components/flyingpigeon/config/canarie-api/canarie_api_monitoring.py.template @@ -24,7 +24,7 @@ SERVICES['flyingpigeon'] = { 'releasenotes': 'https://github.com/bird-house/flyingpigeon/blob/master/CHANGES.rst', 'support': 'https://github.com/bird-house/flyingpigeon/issues', 'source': 'https://github.com/bird-house/flyingpigeon', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/flyingpigeon/wps?service=WPS&version=1.0.0&request=GetCapabilities', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/flyingpigeon/wps?service=WPS&version=1.0.0&request=GetCapabilities', 'licence': 'https://github.com/bird-house/flyingpigeon/blob/master/LICENSE.txt', 'provenance': 'https://github.com/bird-house/flyingpigeon' }, diff --git a/birdhouse/deprecated-components/flyingpigeon/service-config.json.template b/birdhouse/deprecated-components/flyingpigeon/service-config.json.template index 2924e081b..2ff81e734 100644 --- a/birdhouse/deprecated-components/flyingpigeon/service-config.json.template +++ b/birdhouse/deprecated-components/flyingpigeon/service-config.json.template @@ -14,7 +14,7 @@ { "rel": "service", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon?service=WPS&request=GetCapabilities" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon?service=WPS&request=GetCapabilities" }, { "rel": "service-doc", @@ -24,7 +24,7 @@ { "rel": "service-desc", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon?service=WPS&request=GetCapabilities" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon?service=WPS&request=GetCapabilities" }, { "rel": "service-meta", diff --git a/birdhouse/deprecated-components/flyingpigeon/wps.cfg.template b/birdhouse/deprecated-components/flyingpigeon/wps.cfg.template index ff15cd061..218f1caa5 100644 --- a/birdhouse/deprecated-components/flyingpigeon/wps.cfg.template +++ b/birdhouse/deprecated-components/flyingpigeon/wps.cfg.template @@ -1,5 +1,5 @@ [server] -outputurl = https://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs/flyingpigeon +outputurl = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs/flyingpigeon outputpath = /data/wpsoutputs/flyingpigeon maxsingleinputsize = 2097152000.0 diff --git a/birdhouse/deprecated-components/frontend/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/deprecated-components/frontend/config/canarie-api/canarie_api_monitoring.py.template index 7a0a2175c..4b83a1ede 100644 --- a/birdhouse/deprecated-components/frontend/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/deprecated-components/frontend/config/canarie-api/canarie_api_monitoring.py.template @@ -1,5 +1,5 @@ PLATFORMS['server']['monitoring']['Frontend'] = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}' } } diff --git a/birdhouse/deprecated-components/frontend/frontend.env.template b/birdhouse/deprecated-components/frontend/frontend.env.template index e588a127b..1507df2d6 100644 --- a/birdhouse/deprecated-components/frontend/frontend.env.template +++ b/birdhouse/deprecated-components/frontend/frontend.env.template @@ -3,6 +3,6 @@ PAVICS_FRONTEND_PORT=443 PAVICS_FRONTEND_PROTO=https BIRDHOUSE_HOST=${BIRDHOUSE_FQDN} NODE_TLS_REJECT_UNAUTHORIZED=0 -NCWMS_HOST=https://${BIRDHOUSE_FQDN}${TWITCHER_PROTECTED_PATH}/ncWMS2/wms -CATALOG_HOST=https://${BIRDHOUSE_FQDN}${TWITCHER_PROTECTED_PATH}/catalog/pywps -MALLEEFOWL_HOST=https://${BIRDHOUSE_FQDN}${TWITCHER_PROTECTED_PATH}/malleefowl/wps +NCWMS_HOST=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}${TWITCHER_PROTECTED_PATH}/ncWMS2/wms +CATALOG_HOST=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}${TWITCHER_PROTECTED_PATH}/catalog/pywps +MALLEEFOWL_HOST=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}${TWITCHER_PROTECTED_PATH}/malleefowl/wps diff --git a/birdhouse/deprecated-components/malleefowl/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/deprecated-components/malleefowl/config/canarie-api/canarie_api_monitoring.py.template index 7ce8ef425..5c93687de 100644 --- a/birdhouse/deprecated-components/malleefowl/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/deprecated-components/malleefowl/config/canarie-api/canarie_api_monitoring.py.template @@ -19,7 +19,7 @@ SERVICES['Malleefowl'] = { 'releasenotes': 'https://github.com/Ouranosinc/malleefowl/blob/master/CHANGES.rst', 'support': 'https://github.com/Ouranosinc/malleefowl/issues', 'source': 'https://github.com/Ouranosinc/malleefowl', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/malleefowl/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/malleefowl/', 'licence': '', 'provenance': '' }, diff --git a/birdhouse/deprecated-components/phoenix/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/deprecated-components/phoenix/config/canarie-api/canarie_api_monitoring.py.template index 8bd804ceb..51cfbe8ed 100644 --- a/birdhouse/deprecated-components/phoenix/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/deprecated-components/phoenix/config/canarie-api/canarie_api_monitoring.py.template @@ -19,7 +19,7 @@ SERVICES['Phoenix'] = { 'releasenotes': 'https://github.com/ouranosinc/pyramid-phoenix/CHANGES.rst', 'support': 'https://github.com/ouranosinc/pyramid-phoenix/issues', 'source': 'https://github.com/ouranosinc/pyramid-phoenix', - 'tryme': 'https://${BIRDHOUSE_FQDN}:8443/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}:8443/', 'licence': 'https://github.com/ouranosinc/pyramid-phoenix/blob/master/LICENSE.txt', 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' }, @@ -27,7 +27,7 @@ SERVICES['Phoenix'] = { "Phoenix": { 'request': { # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) - 'url': 'https://${BIRDHOUSE_FQDN}:8443/' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}:8443/' } } } diff --git a/birdhouse/deprecated-components/project-api/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/deprecated-components/project-api/config/canarie-api/canarie_api_monitoring.py.template index 9b3b4e7e4..411c2158c 100644 --- a/birdhouse/deprecated-components/project-api/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/deprecated-components/project-api/config/canarie-api/canarie_api_monitoring.py.template @@ -1,6 +1,6 @@ PLATFORMS['server']['monitoring']['Project'] = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/project-api/explorer/' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/project-api/explorer/' } } PLATFORMS['server']['stats']['route'] = '/project-api/.*' diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index 795b5b057..3b643412f 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -541,8 +541,8 @@ export THREDDS_ADDITIONAL_CATALOG='' # To setup Github as login, goto under section [OAuth Apps] # and create a new Magpie application with configurations: # -# Homepage URL: https://${BIRDHOUSE_FQDN} -# Authorization callback URL: https://${BIRDHOUSE_FQDN}/magpie/providers/github/signin +# Homepage URL: ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN} +# Authorization callback URL: ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/magpie/providers/github/signin # # Then, specify obtained Github client ID/Secret for this Magpie OAuth App with following variables. # @@ -593,7 +593,7 @@ export THREDDS_ADDITIONAL_CATALOG='' # Raven to use the local Geoserver instead of the default production. # See raven/default.env for more info. -#export RAVEN_GEO_URL="https://${BIRDHOUSE_FQDN}/geoserver/" +#export RAVEN_GEO_URL="${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/geoserver/" # Mount point on host machine to store mongodb server data # (note: if using 'BIRDHOUSE_DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env') diff --git a/birdhouse/optional-components/README.rst b/birdhouse/optional-components/README.rst index 2210bbe1b..a2ea602c4 100644 --- a/birdhouse/optional-components/README.rst +++ b/birdhouse/optional-components/README.rst @@ -487,3 +487,31 @@ For developers, to create a new parser that can be used to track log files: .. _log-parser: https://github.com/DACCS-Climate/log-parser/ .. _prometheus_python_metrics: https://prometheus.github.io/client_python/instrumenting/ + +.. _local-dev-test + +Local Dev Test +-------------- + +This allows users to deploy the entire stack locally for development or testing purposes. + +If this component is enabled the following configuration settings must also be set in the local environment file: + + * ``export BIRDHOUSE_FQDN=host.docker.internal`` + * ``export BIRDHOUSE_HTTP_ONLY=True`` + +You should also add ``host.docker.internal`` to your ``/etc/hosts`` file pointing to the loopback address so that URLs +generated by Birdhouse that refer to ``host.docker.internal`` will resolve properly in a browser: + +.. code:: shell + + echo '127.0.0.1 host.docker.internal' | sudo tee -a /etc/hosts + +After deploying the stack, you can now interact with the Birdhouse software at ``http://host.docker.internal`` from the +machine that is the docker host. + +Note that you do *not* need an SSL certificate set up to deploy the stack in this way. + +.. warning:: + + **DO NOT** enable this component in production. This is intended for local development and test purposes only! diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/a_demo_override_precedence.py b/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/a_demo_override_precedence.py index b3c238cf6..31b503d0a 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/a_demo_override_precedence.py +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/a_demo_override_precedence.py @@ -5,7 +5,7 @@ # SERVICES['node']['monitoring'].update({ # 'Thredds-public': { # 'request': { -# 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/toto', +# 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/toto', # }, # }, # }) diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/z_demo_only_py_file_are_loaded.wrongsuffix b/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/z_demo_only_py_file_are_loaded.wrongsuffix index a85d12907..5e2d072dd 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/z_demo_only_py_file_are_loaded.wrongsuffix +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/z_demo_only_py_file_are_loaded.wrongsuffix @@ -2,7 +2,7 @@ SERVICES['node']['monitoring'].update({ 'Thredds-public': { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/wrong-suffix', + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/wrong-suffix', }, }, }) diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/catalog/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/catalog/canarie_api_full_monitoring.py.template index dc8296d8b..2241fbb32 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/catalog/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/catalog/canarie_api_full_monitoring.py.template @@ -3,7 +3,7 @@ import copy service_public = "Catalog-public" service_public_cfg = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/catalog?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/catalog?service=WPS&version=1.0.0&request=GetCapabilities' }, } diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/cowbird/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/cowbird/canarie_api_full_monitoring.py.template index 1e8596a70..58843b316 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/cowbird/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/cowbird/canarie_api_full_monitoring.py.template @@ -4,7 +4,7 @@ if "Cowbird" in SERVICES: # See notes in 'components/cowbird/docker-compose-extra.yml' # about the conditional loading of this config in 'proxy' service. cowbird_cfg = copy.deepcopy(SERVICES["Cowbird"]["monitoring"]["Cowbird"]) - cowbird_cfg["request"]["url"] = "https://${BIRDHOUSE_FQDN_PUBLIC}/cowbird/" + cowbird_cfg["request"]["url"] = "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/cowbird/" SERVICES["Cowbird"]["monitoring"]["Cowbird-public"] = cowbird_cfg # vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 syntax=python diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/finch/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/finch/canarie_api_full_monitoring.py.template index 89b2066a8..b3e84fa7f 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/finch/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/finch/canarie_api_full_monitoring.py.template @@ -3,7 +3,7 @@ import copy service_public = "Finch-public" service_public_cfg = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&version=1.0.0&request=GetCapabilities' } } diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/hummingbird/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/hummingbird/canarie_api_full_monitoring.py.template index 9062bd33a..6258d88c6 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/hummingbird/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/hummingbird/canarie_api_full_monitoring.py.template @@ -3,7 +3,7 @@ import copy service_public = "Hummingbird-public" service_public_cfg = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&version=1.0.0&request=GetCapabilities' } } diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/malleefowl/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/malleefowl/canarie_api_full_monitoring.py.template index 8db3c6fbc..91aeedb35 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/malleefowl/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/malleefowl/canarie_api_full_monitoring.py.template @@ -3,7 +3,7 @@ import copy service_public = "Malleefowl-public" service_public_cfg = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/malleefowl?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/malleefowl?service=WPS&version=1.0.0&request=GetCapabilities' } } diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/ncwms2/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/ncwms2/canarie_api_full_monitoring.py.template index 3fb67f44a..9c81ec1f7 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/ncwms2/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/ncwms2/canarie_api_full_monitoring.py.template @@ -3,7 +3,7 @@ import copy service_public = "ncWMS2-public" service_public_cfg = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/ncWMS2/wms?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/ncWMS2/wms?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0' } } diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/raven/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/raven/canarie_api_full_monitoring.py.template index 3420fa0dc..09fcce609 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/raven/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/raven/canarie_api_full_monitoring.py.template @@ -3,7 +3,7 @@ import copy service_public = "Raven-public" service_public_cfg = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&version=1.0.0&request=GetCapabilities' } } diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/thredds/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/thredds/canarie_api_full_monitoring.py.template index 03f44ab94..a5ae7bda1 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/thredds/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/thredds/canarie_api_full_monitoring.py.template @@ -3,7 +3,7 @@ import copy service_public = "Thredds-public" service_public_cfg = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/thredds/catalog.html', + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/thredds/catalog.html', }, } diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/weaver/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/weaver/canarie_api_full_monitoring.py.template index 12ea37be9..717093961 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/weaver/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/weaver/canarie_api_full_monitoring.py.template @@ -4,7 +4,7 @@ if "Weaver" in SERVICES: # See notes in 'components/weaver/docker-compose-extra.yml' # about the conditional loading of this config in 'proxy' service. weaver_cfg = copy.deepcopy(SERVICES["Weaver"]["monitoring"]["Weaver"]) - weaver_cfg["request"]["url"] = "https://${BIRDHOUSE_FQDN_PUBLIC}/weaver/" + weaver_cfg["request"]["url"] = "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/weaver/" SERVICES["Weaver"]["monitoring"]["Weaver-public"] = weaver_cfg # vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 syntax=python diff --git a/birdhouse/optional-components/emu/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/optional-components/emu/config/canarie-api/canarie_api_monitoring.py.template index 2b822d1b6..8f3b9b26f 100644 --- a/birdhouse/optional-components/emu/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/optional-components/emu/config/canarie-api/canarie_api_monitoring.py.template @@ -1,7 +1,7 @@ SERVICES['node']['monitoring'].update({ '${EMU_NAME}-public': { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${EMU_NAME}?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${EMU_NAME}?service=WPS&version=1.0.0&request=GetCapabilities' }, }, '${EMU_NAME}': { diff --git a/birdhouse/optional-components/emu/wps.cfg.template b/birdhouse/optional-components/emu/wps.cfg.template index 53001b15e..b9b9738c3 100644 --- a/birdhouse/optional-components/emu/wps.cfg.template +++ b/birdhouse/optional-components/emu/wps.cfg.template @@ -1,5 +1,5 @@ [server] -outputurl = https://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs +outputurl = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs outputpath = /data/wpsoutputs [logging] diff --git a/birdhouse/optional-components/generic_bird/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/optional-components/generic_bird/config/canarie-api/canarie_api_monitoring.py.template index c71b88ef5..6e12b859c 100644 --- a/birdhouse/optional-components/generic_bird/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/optional-components/generic_bird/config/canarie-api/canarie_api_monitoring.py.template @@ -1,7 +1,7 @@ SERVICES['node']['monitoring'].update({ '${GENERIC_BIRD_NAME}-public': { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${GENERIC_BIRD_NAME}?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${GENERIC_BIRD_NAME}?service=WPS&version=1.0.0&request=GetCapabilities' }, }, '${GENERIC_BIRD_NAME}': { diff --git a/birdhouse/optional-components/generic_bird/wps.cfg.template b/birdhouse/optional-components/generic_bird/wps.cfg.template index 9a029e823..076881c47 100644 --- a/birdhouse/optional-components/generic_bird/wps.cfg.template +++ b/birdhouse/optional-components/generic_bird/wps.cfg.template @@ -1,5 +1,5 @@ [server] -outputurl = https://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs +outputurl = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs outputpath = /data/wpsoutputs # default 3mb, fix "Broken pipe" between the proxy and the wps service diff --git a/birdhouse/optional-components/local-dev-test/.gitignore b/birdhouse/optional-components/local-dev-test/.gitignore new file mode 100644 index 000000000..9b6cadaa8 --- /dev/null +++ b/birdhouse/optional-components/local-dev-test/.gitignore @@ -0,0 +1 @@ +docker-compose-extra-ignore.yml diff --git a/birdhouse/optional-components/local-dev-test/default.env b/birdhouse/optional-components/local-dev-test/default.env new file mode 100644 index 000000000..4b9f67359 --- /dev/null +++ b/birdhouse/optional-components/local-dev-test/default.env @@ -0,0 +1,2 @@ +export JUPYTERHUB_DOCKER_EXTRA_HOSTS='host.docker.internal:host-gateway' +export NODE_EXPORTER_HOST_BIND_PROPOGATION= diff --git a/birdhouse/optional-components/local-dev-test/pre-docker-compose-up.include b/birdhouse/optional-components/local-dev-test/pre-docker-compose-up.include new file mode 100644 index 000000000..07830e752 --- /dev/null +++ b/birdhouse/optional-components/local-dev-test/pre-docker-compose-up.include @@ -0,0 +1,29 @@ +#!/usr/bin/env sh + +error= +if [ "${BIRDHOUSE_FQDN}" != "host.docker.internal" ]; then + log ERROR 'BIRDHOUSE_FQDN should be set to 'host.docker.internal' when the local-dev-test optional component is enabled' + error=True +fi + +if [ "${BIRDHOUSE_HTTP_ONLY}" != "True" ]; then + log ERROR 'BIRDHOUSE_HTTP_ONLY should be set to 'True' when the local-dev-test optional component is enabled' + error=True +fi + +[ "$error" = 'True' ] && exit 1 + +# Note: filename is not docker-compose-extra.yml so that it won't get added prematurely to COMPOSE_CONF_LIST +THIS_COMPOSE_FILE="${COMPOSE_DIR}/optional-components/local-dev-test/docker-compose-extra-ignore.yml" +echo 'version: "3.4"' > "${THIS_COMPOSE_FILE}" +echo "services:" >> "${THIS_COMPOSE_FILE}" + +for service in $(PROXY_HTTP_PORT=80 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} config --services); do + printf ' %s:\n extra_hosts:\n - "host.docker.internal:host-gateway"\n' $service >> "${THIS_COMPOSE_FILE}" +done + +COMPOSE_CONF_LIST="${COMPOSE_CONF_LIST} -f ${THIS_COMPOSE_FILE}" + +log INFO "adding ${THIS_COMPOSE_FILE} to COMPOSE_CONF_LIST" + +unset THIS_COMPOSE_FILE diff --git a/birdhouse/optional-components/secure-data-proxy/config/proxy/conf.extra-service.d/secure-data-auth.conf.template b/birdhouse/optional-components/secure-data-proxy/config/proxy/conf.extra-service.d/secure-data-auth.conf.template index 8034910c3..333f68d05 100644 --- a/birdhouse/optional-components/secure-data-proxy/config/proxy/conf.extra-service.d/secure-data-auth.conf.template +++ b/birdhouse/optional-components/secure-data-proxy/config/proxy/conf.extra-service.d/secure-data-auth.conf.template @@ -4,7 +4,7 @@ # note: using 'TWITCHER_VERIFY_PATH' path to avoid performing the request via 'proxy' endpoint # This ensures that the data access is validated for the user, but does not trigger its access/download twice. # Also, avoids getting an error as 'secure-data-proxy' private URL in Magpie doesn't resolve to a valid path. - proxy_pass https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/secure-data-proxy$request_uri; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/secure-data-proxy$request_uri; proxy_pass_request_body off; proxy_set_header Host $host; proxy_set_header Content-Length ""; diff --git a/birdhouse/optional-components/test-cowbird-jupyter-access/docker-compose-extra.yml b/birdhouse/optional-components/test-cowbird-jupyter-access/docker-compose-extra.yml index e32c111cb..50a444162 100644 --- a/birdhouse/optional-components/test-cowbird-jupyter-access/docker-compose-extra.yml +++ b/birdhouse/optional-components/test-cowbird-jupyter-access/docker-compose-extra.yml @@ -9,7 +9,7 @@ services: environment: MAGPIE_ADMIN_USERNAME: ${MAGPIE_ADMIN_USERNAME} MAGPIE_ADMIN_PASSWORD: ${MAGPIE_ADMIN_PASSWORD} - BIRDHOUSE_HOST_URL: https://${BIRDHOUSE_FQDN_PUBLIC} + BIRDHOUSE_HOST_URL: ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC} WPS_OUTPUTS_DIR: ${BIRDHOUSE_WPS_OUTPUTS_DIR} WORKSPACE_DIR: ${BIRDHOUSE_DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES} TEST_COWBIRD_JUPYTERHUB_USERNAME: ${TEST_COWBIRD_JUPYTERHUB_USERNAME} diff --git a/birdhouse/optional-components/test-weaver/config/weaver/request_options.yml.template b/birdhouse/optional-components/test-weaver/config/weaver/request_options.yml.template index f4434b3b7..1794dd160 100644 --- a/birdhouse/optional-components/test-weaver/config/weaver/request_options.yml.template +++ b/birdhouse/optional-components/test-weaver/config/weaver/request_options.yml.template @@ -11,5 +11,5 @@ requests: # disable SSL verification for test instance using self-signed certificate # avoid doing this on real instance to keep it secure against man-in-the-middle attacks - - url: https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/weaver/providers/hummingbird/processes/ncdump/jobs + - url: ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/weaver/providers/hummingbird/processes/ncdump/jobs verify: false diff --git a/birdhouse/optional-components/testthredds/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/optional-components/testthredds/config/canarie-api/canarie_api_monitoring.py.template index 9c0051f35..9bbfc5879 100644 --- a/birdhouse/optional-components/testthredds/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/optional-components/testthredds/config/canarie-api/canarie_api_monitoring.py.template @@ -1,7 +1,7 @@ SERVICES['node']['monitoring'].update({ '${TESTTHREDDS_NAME}-public': { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/${TESTTHREDDS_CONTEXT_ROOT}/catalog.html' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/${TESTTHREDDS_CONTEXT_ROOT}/catalog.html' }, }, '${TESTTHREDDS_NAME}': { diff --git a/birdhouse/read-configs.include.sh b/birdhouse/read-configs.include.sh index bad87dcfa..64e337c30 100644 --- a/birdhouse/read-configs.include.sh +++ b/birdhouse/read-configs.include.sh @@ -103,7 +103,7 @@ read_default_env() { . "${COMPOSE_DIR}/default.env" else - log WARN "'${COMPOSE_DIR}/default.env' not found" 1>&2 + log WARN "'${COMPOSE_DIR}/default.env' not found" fi } @@ -123,7 +123,7 @@ read_env_local() { eval "${saved_shell_options}" else - log WARN "'${BIRDHOUSE_LOCAL_ENV}' not found" 1>&2 + log WARN "'${BIRDHOUSE_LOCAL_ENV}' not found" fi } @@ -155,7 +155,7 @@ source_conf_files() { # corresponding PR are merged and old component names can be removed # after the corresponding PR are merge without any impact on the # autodeploy process. - log WARN "'${adir}' in ${conf_locations} does not exist" 1>&2 + log WARN "'${adir}' in ${conf_locations} does not exist" fi if [ -f "${adir}/default.env" ]; then # Source config settings of dependencies first if they haven't been sourced previously. diff --git a/birdhouse/scripts/check-instance-ready b/birdhouse/scripts/check-instance-ready index 122bd6924..e4f872353 100755 --- a/birdhouse/scripts/check-instance-ready +++ b/birdhouse/scripts/check-instance-ready @@ -19,7 +19,7 @@ if [ -f "${COMPOSE_DIR}/read-configs.include.sh" ]; then fi set -x -curl --include --silent "https://${BIRDHOUSE_FQDN}/canarie/node/service/stats" | head +curl --include --silent "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/canarie/node/service/stats" | head set +x echo " @@ -28,7 +28,7 @@ The curl above should return the HTTP response code 200 to confirm instance is r set -x HTTP_RESPONSE_CODE="$( \ - curl --write-out '%{http_code}' --output /dev/null --silent "https://${BIRDHOUSE_FQDN}/canarie/node/service/stats" \ + curl --write-out '%{http_code}' --output /dev/null --silent "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/canarie/node/service/stats" \ )" if [ "${HTTP_RESPONSE_CODE}" -ne 200 ]; then set +x @@ -41,5 +41,5 @@ Will retry only once more and exit immediately. " set -x sleep 65 - curl --include --silent "https://${BIRDHOUSE_FQDN}/canarie/node/service/stats" | head + curl --include --silent "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/canarie/node/service/stats" | head fi diff --git a/birdhouse/scripts/clear-running-wps-jobs-in-db.sh b/birdhouse/scripts/clear-running-wps-jobs-in-db.sh index 897e3deb1..1cdfae0ef 100755 --- a/birdhouse/scripts/clear-running-wps-jobs-in-db.sh +++ b/birdhouse/scripts/clear-running-wps-jobs-in-db.sh @@ -11,7 +11,7 @@ fi # eg: DB_NAME=finch DB_NAME="$1" if [ -z "$DB_NAME" ]; then - log ERROR "please provide a database name, ex: finch" 1>&2 + log ERROR "please provide a database name, ex: finch" exit 2 fi shift diff --git a/birdhouse/scripts/logging.include.sh b/birdhouse/scripts/logging.include.sh index fcce426bd..6129117bf 100644 --- a/birdhouse/scripts/logging.include.sh +++ b/birdhouse/scripts/logging.include.sh @@ -14,6 +14,13 @@ if [ "${BIRDHOUSE_COLOR}" -eq "1" ]; then fi BIRDHOUSE_LOG_LEVEL=${BIRDHOUSE_LOG_LEVEL:-INFO} +BIRDHOUSE_LOG_FD=${BIRDHOUSE_LOG_FD:-2} +if [ "${__BIRDHOUSE_SUPPORTED_INTERFACE}" != 'True' ]; then + # logs were previously written to stdout for DEBUG and INFO + # logs were previously intended to be written to stderr for WARN, ERROR, and CRITICAL + # (this supports backwards compatible scripts that don't use the interface) + BIRDHOUSE_LOG_DEST_OVERRIDE=${BIRDHOUSE_LOG_DEST_OVERRIDE:-"DEBUG:fd:1:INFO:fd:1:WARN:fd:2:ERROR:fd:2:CRITICAL:fd:2"} +fi export LOG_DEBUG="${GRAY}DEBUG${NORMAL}: " export LOG_INFO="${BLUE}INFO${NORMAL}: " export LOG_WARN="${YELLOW}WARNING${NORMAL}: " @@ -21,41 +28,128 @@ export LOG_ERROR="${RED}ERROR${NORMAL}: " export LOG_CRITICAL="${REG_BG_BOLD}CRITICAL${NORMAL}: " # to report misuse of functions -# Usage: log {LEVEL} "{message}" [...] -# Any amount of messages can be passed to the function. -log() { - if [ "${BIRDHOUSE_LOG_LEVEL}" != DEBUG ] \ - && [ "${BIRDHOUSE_LOG_LEVEL}" != INFO ] \ - && [ "${BIRDHOUSE_LOG_LEVEL}" != WARN ] \ - && [ "${BIRDHOUSE_LOG_LEVEL}" != ERROR ]; then - echo "${LOG_CRITICAL}Invalid log level setting: [BIRDHOUSE_LOG_LEVEL=${BIRDHOUSE_LOG_LEVEL}]." - exit 2 +# Determines where to send log messages: +# - to the file descriptor set by BIRDHOUSE_LOG_FD, or if there is a "fd" option in BIRDHOUSE_LOG_DEST_OVERRIDE for the given log level +# - to the file set by BIRDHOUSE_LOG_FILE, or if there is a "file" option in BIRDHOUSE_LOG_DEST_OVERRIDE for the given log level +# - suppresses writing to a file descriptor if BIRDHOUSE_LOG_QUIET is "True" or if there is a "quiet" option in BIRDHOUSE_LOG_DEST_OVERRIDE for the given log level +# +# The BIRDHOUSE_LOG_DEST_OVERRIDE contains a ':' delimited string that determines how to override the log destination for specific log levels. +# BIRDHOUSE_LOG_DEST_OVERRIDE sections contain ':