docker-compose.yml

# Service configurations for all microservices
# has to be specified by environment files

version: '3.7'

services:

  # ----------------- UI SERVICE (/ui) --------------------
  ui:
    image: ${DOCKER_REGISTRY}/ui
    build:
      context: ./ui/
      args:
        UI_BASE_URL: /
        ADAPTER_SERVICE_URL: /api/adapter
        STORAGE_SERVICE_URL: /api/storage
        PIPELINE_SERVICE_URL: /api/pipelines
        NOTIFICATION_SERVICE_URL: /api/notification
        SCHEMA_SERVICE_URL: /api/schema
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.to-ui.rule=PathPrefix(`/`)"

  # ----------------- ADAPTER SERVICE (/adapter) --------------------
  adapter:
    image: ${DOCKER_REGISTRY}/adapter
    build:
      context: ./adapter/
    depends_on:
      - rabbitmq
      - adapter-db
      - adapter-outboxer
    environment:
      SPRING_RABBITMQ_HOST: rabbitmq
      SPRING_RABBITMQ_PORT: 5672
      SPRING_RABBITMQ_USERNAME: rabbit_adm
      SPRING_RABBITMQ_PASSWORD: R4bb!7_4DM_p4SS
      SPRING_DATASOURCE_URL: jdbc:postgresql://adapter-db:5432/adapterservice
      SPRING_DATASOURCE_USERNAME: adapterservice
      SPRING_DATASOURCE_PASSWORD: admin

      ADAPTER_AMQP_IMPORT_SUCCESS_TOPIC: 'datasource.execution.success'
      ADAPTER_AMQP_IMPORT_FAILED_TOPIC: 'datasource.execution.failed'
      ADAPTER_AMQP_DATASOURCE_CREATED_TOPIC: 'datasource.config.created'
      ADAPTER_AMQP_DATASOURCE_UPDATED_TOPIC: 'datasource.config.updated'
      ADAPTER_AMQP_DATASOURCE_DELETED_TOPIC: 'datasource.config.deleted'

      ADAPTER_AMQP_ADAPTER_EXCHANGE: 'ods_global'
      ADAPTER_AMQP_DATASOURCE_IMPORT_TRIGGER_QUEUE: 'adapter.datasource-import-trigger'
      ADAPTER_AMQP_DATASOURCE_IMPORT_TRIGGER_QUEUE_TOPIC: 'datasource.import-trigger.*'
      ADAPTER_AMQP_DATASOURCE_IMPORT_TRIGGER_CREATED_TOPIC: 'datasource.import-trigger.created'
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.to-adapter.rule=PathPrefix(`/api/adapter`)"
      - "traefik.http.routers.to-adapter.middlewares=adapter-stripprefix@docker"
      - "traefik.http.middlewares.adapter-stripprefix.stripprefix.prefixes=/api/adapter"

  adapter-db:
    image: postgres:13-alpine
    environment:
      - POSTGRES_USER=adapterservice
      - POSTGRES_PASSWORD=admin
    command:
      # This enables and configures Postgres logical decoding feature that is needed for outboxer/debezium to work
      - '-cwal_level=logical'
      - '-cmax_wal_senders=1'
      - '-cmax_replication_slots=1'
    # Uncomment if you want to access to adapter-db from outside the docker environment
    # ports:
    #   - 5432:5432

  adapter-outboxer:
    image: ghcr.io/jvalue/outboxer-postgres2rabbitmq
    environment:
      - OUTBOXER_DATABASE_HOSTNAME=adapter-db
      - OUTBOXER_DATABASE_PORT=5432
      - OUTBOXER_DATABASE_USER=adapterservice
      - OUTBOXER_DATABASE_PASSWORD=admin
      - OUTBOXER_DATABASE_DBNAME=adapterservice
      - OUTBOXER_DATABASE_SERVER_NAME=adapter-outboxer
      - OUTBOXER_PUBLISHER_AMQP_URL=amqp://rabbit_adm:R4bb!7_4DM_p4SS@rabbitmq:5672
      - OUTBOXER_PUBLISHER_AMQP_EXCHANGE=ods_global
      - OUTBOXER_PUBLISHER_AMQP_RETRIES=50
      - OUTBOXER_PUBLISHER_AMQP_RETRY_DELAY_MS=1000
      # In production this file should be storage outside of the container, so it does survive outboxer restarts.
      #- OUTBOXER_OFFSET_STORAGE_FILE_FILENAME=/data/offset.dat
    depends_on:
      - adapter-db
      - rabbitmq

  # ----------------- SCHEDULER SERVICE (/scheduler) --------------------
  scheduler:
    image: ${DOCKER_REGISTRY}/scheduler
    environment:
      MAX_TRIGGER_RETRIES: 2
      ADAPTER_SERVICE_URL: http://adapter:8080
      CONNECTION_RETRIES: 20
      CONNECTION_BACKOFF_IN_MS: 5000

      POSTGRES_HOST: "scheduler-db"
      POSTGRES_PORT: 5432
      POSTGRES_USER: "scheduler-service"
      POSTGRES_PW: "pw"
      POSTGRES_DB: "ods-schedulers"
      POSTGRES_SCHEMA: "public" # other schemas have to be created somewhere before use!
      POSTGRES_SSL: "false"

      AMQP_URL: 'amqp://rabbit_adm:R4bb!7_4DM_p4SS@rabbitmq:5672'
      AMQP_SCHEDULER_EXCHANGE: 'ods_global'
      AMQP_SCHEDULER_QUEUE: 'scheduler.datasource-config'
      AMQP_DATASOURCE_CONFIG_TOPIC: 'datasource.config.*'
      AMQP_DATASOURCE_CONFIG_CREATED_TOPIC: 'datasource.config.created'
      AMQP_DATASOURCE_CONFIG_UPDATED_TOPIC: 'datasource.config.updated'
      AMQP_DATASOURCE_CONFIG_DELETED_TOPIC: 'datasource.config.deleted'
      AMQP_DATASOURCE_IMPORT_TRIGGER_TOPIC: 'datasource.import-trigger.*'
      AMQP_DATASOURCE_IMPORT_TRIGGER_CREATED_TOPIC: 'datasource.import-trigger.created'
    build:
      context: ./scheduler/
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.to-scheduler.rule=PathPrefix(`/api/scheduler`)"
      - "traefik.http.routers.to-scheduler.middlewares=scheduler-stripprefix@docker"
      - "traefik.http.middlewares.scheduler-stripprefix.stripprefix.prefixes=/api/scheduler"
    depends_on:
      - rabbitmq
      - scheduler-db
      - scheduler-outboxer

  scheduler-db:
    image: postgres:13-alpine
    environment:
      POSTGRES_USER: 'scheduler-service'
      POSTGRES_PASSWORD: 'pw'
      POSTGRES_DB: 'ods-schedulers'
    command:
      # This enables and configures Postgres logical decoding feature that is needed for outboxer/debezium to work
      - '-cwal_level=logical'
      - '-cmax_wal_senders=1'
      - '-cmax_replication_slots=1'

  scheduler-outboxer:
    image: ghcr.io/jvalue/outboxer-postgres2rabbitmq
    environment:
      - OUTBOXER_DATABASE_HOSTNAME=scheduler-db
      - OUTBOXER_DATABASE_PORT=5432
      - OUTBOXER_DATABASE_USER=scheduler-service
      - OUTBOXER_DATABASE_PASSWORD=pw
      - OUTBOXER_DATABASE_DBNAME=ods-schedulers
      - OUTBOXER_DATABASE_SERVER_NAME=scheduler-outboxer
      - OUTBOXER_PUBLISHER_AMQP_URL=amqp://rabbit_adm:R4bb!7_4DM_p4SS@rabbitmq:5672
      - OUTBOXER_PUBLISHER_AMQP_EXCHANGE=ods_global
      - OUTBOXER_PUBLISHER_AMQP_RETRIES=5
      - OUTBOXER_PUBLISHER_AMQP_RETRY_DELAY_MS=1000
      # In production this file should be storage outside of the container, so it does survive outboxer restarts.
      #- OUTBOXER_OFFSET_STORAGE_FILE_FILENAME=/data/offset.dat
    depends_on:
      - scheduler-db
      - rabbitmq
  # ----------------- STORAGE SERVICE (/storage) --------------------

  storage-mq:
    image: ${DOCKER_REGISTRY}/storage-mq
    build: ./storage/storage-mq/
    environment:
      CONNECTION_RETRIES: 30
      CONNECTION_BACKOFF_IN_MS: 2000

      POSTGRES_HOST: storage-db
      POSTGRES_PORT: 5432
      POSTGRES_DB: ods
      POSTGRES_USER: storagemq
      POSTGRES_PW: storagemq-pw
      POSTGRES_SCHEMA: 'storage'
      POSTGRES_SSL: "false"

      AMQP_URL: 'amqp://rabbit_adm:R4bb!7_4DM_p4SS@rabbitmq:5672'

      AMQP_PIPELINE_CONFIG_EXCHANGE: 'ods_global'
      AMQP_PIPELINE_CONFIG_QUEUE: 'storage-mq.pipeline-config'
      AMQP_PIPELINE_CONFIG_QUEUE_TOPIC: 'pipeline.config.*'
      AMQP_PIPELINE_CONFIG_CREATED_TOPIC: 'pipeline.config.created'
      AMQP_PIPELINE_CONFIG_DELETED_TOPIC: 'pipeline.config.deleted'

      AMQP_PIPELINE_EXECUTION_EXCHANGE: 'ods_global'
      AMQP_PIPELINE_EXECUTION_QUEUE: 'storage-mq.pipeline-execution'
      AMQP_PIPELINE_EXECUTION_QUEUE_TOPIC: 'pipeline.execution.success'
      AMQP_PIPELINE_EXECUTION_SUCCESS_TOPIC: 'pipeline.execution.success'
    depends_on:
      - storage-db
      - storage-db-liquibase # TODO: make sure the migration runs through before reading db structure
      - rabbitmq
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.to-storage-mq.rule=PathPrefix(`/api/storage-mq`)"
      - "traefik.http.routers.to-storage-mq.middlewares=storage-mq-stripprefix@docker"
      - "traefik.http.middlewares.storage-mq-stripprefix.stripprefix.prefixes=/api/storage-mq"

  storage: # Wraps postgres database with API
    image: ${DOCKER_REGISTRY}/storage
    build: ./storage/postgrest/
    environment:
      DATABASE_HOST: storage-db
      DATABASE_PORT: 5432
      DATABASE_NAME: ods
      DATABASE_USER: postgrest
      DATABASE_PW: postgres-wrapper-pw
      DATABASE_ANON: web_anon
    depends_on:
      - storage-db
      - storage-db-liquibase # TODO: make sure the migration runs through before reading db structure
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.to-storage.rule=Path(`/api/storage`) || PathPrefix(`/api/storage/`)"
      - "traefik.http.routers.to-storage.middlewares=storage-stripprefix@docker,storage-addaccept@docker"
      - "traefik.http.middlewares.storage-stripprefix.stripprefix.prefixes=/api/storage"
      - "traefik.http.middlewares.storage-addaccept.headers.customrequestheaders.Accept=application/json" # Firefox Browser Support

  storage-db:
    image: postgres:13-alpine
    environment:
      POSTGRES_DB: ods
      POSTGRES_USER: ods_admin
      POSTGRES_PASSWORD: ods_pw
  # Uncomment this if you want to persist the data.
  # volumes:
  #   - "./pgdata:/var/lib/postgresql/data"

  # Uncomment this if you want to manage the Postgres databases with adminer
  # adminer: # management UI for Postgres
  #   image: adminer
  #   ports:
  #     - 8081:8080

  storage-db-liquibase: # perform database migration on start up
    image: ${DOCKER_REGISTRY}/storage-db-liquibase
    build: ./storage/liquibase/ # TODO: Sql Injection is currently
    environment:
      DATABASE_HOST: storage-db
      DATABASE_PORT: 5432
      DATABASE_NAME: ods
      DATABASE_USER: ods_admin
      DATABASE_PW: ods_pw
      CONNECTION_RETRIES: 5
      CONNECTION_BACKOFF_IN_SECONDS: 2
    depends_on:
      - storage-db

  # ----------------- PIPELINE SERVICE (/pipelines) --------------------
  pipeline:
    image: ${DOCKER_REGISTRY}/pipeline
    build:
      context: ./pipeline/
    depends_on:
      - rabbitmq
      - pipeline-db
      - pipeline-outboxer
    environment:
      CONNECTION_RETRIES: '30'
      CONNECTION_BACKOFF_IN_MS: '2000'

      POSTGRES_HOST: "pipeline-db"
      POSTGRES_PORT: 5432
      POSTGRES_USER: "pipeline-service"
      POSTGRES_PW: "pw"
      POSTGRES_DB: "ods-pipelines"
      POSTGRES_SCHEMA: "public" # other schemas have to be created somewhere before use!
      POSTGRES_SSL: "false"

      AMQP_URL: "amqp://rabbit_adm:R4bb!7_4DM_p4SS@rabbitmq:5672"

      AMQP_PIPELINE_EXECUTION_SUCCESS_TOPIC: 'pipeline.execution.success'
      AMQP_PIPELINE_EXECUTION_ERROR_TOPIC: 'pipeline.execution.error'
      AMQP_PIPELINE_CONFIG_CREATED_TOPIC: 'pipeline.config.created'
      AMQP_PIPELINE_CONFIG_UPDATED_TOPIC: 'pipeline.config.updated'
      AMQP_PIPELINE_CONFIG_DELETED_TOPIC: 'pipeline.config.deleted'

      AMQP_DATASOURCE_EXECUTION_EXCHANGE: 'ods_global'
      AMQP_DATASOURCE_EXECUTION_QUEUE: 'pipeline.datasource-execution'
      AMQP_DATASOURCE_EXECUTION_QUEUE_TOPIC: 'datasource.execution.success'
      AMQP_DATASOURCE_EXECUTION_SUCCESS_TOPIC: 'datasource.execution.success'
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.to-pipeline.rule=PathPrefix(`/api/pipelines`)"
      - "traefik.http.routers.to-pipeline.middlewares=pipeline-stripprefix@docker"
      - "traefik.http.middlewares.pipeline-stripprefix.stripprefix.prefixes=/api/pipelines"

  pipeline-db:
    image: postgres:13-alpine
    environment:
      POSTGRES_USER: 'pipeline-service'
      POSTGRES_PASSWORD: 'pw'
      POSTGRES_DB: 'ods-pipelines'
    command:
      # This enables and configures Postgres logical decoding feature that is needed for outboxer/debezium to work
      - '-cwal_level=logical'
      - '-cmax_wal_senders=1'
      - '-cmax_replication_slots=1'

  pipeline-outboxer:
    image: ghcr.io/jvalue/outboxer-postgres2rabbitmq
    environment:
      - OUTBOXER_DATABASE_HOSTNAME=pipeline-db
      - OUTBOXER_DATABASE_PORT=5432
      - OUTBOXER_DATABASE_USER=pipeline-service
      - OUTBOXER_DATABASE_PASSWORD=pw
      - OUTBOXER_DATABASE_DBNAME=ods-pipelines
      - OUTBOXER_DATABASE_SERVER_NAME=pipeline-outboxer
      - OUTBOXER_PUBLISHER_AMQP_URL=amqp://rabbit_adm:R4bb!7_4DM_p4SS@rabbitmq:5672
      - OUTBOXER_PUBLISHER_AMQP_EXCHANGE=ods_global
      - OUTBOXER_PUBLISHER_AMQP_RETRIES=5
      - OUTBOXER_PUBLISHER_AMQP_RETRY_DELAY_MS=1000
      # In production this file should be storage outside of the container, so it does survive outboxer restarts.
      #- OUTBOXER_OFFSET_STORAGE_FILE_FILENAME=/data/offset.dat
    depends_on:
      - pipeline-db
      - rabbitmq

  # ----------------- NOTIFICATION SERVICE (/notification) --------------------
  notification:
    image: ${DOCKER_REGISTRY}/notification
    build:
      context: ./notification/
    environment:
      # Postgres connection vars
      POSTGRES_HOST: 'notification-db'
      POSTGRES_PORT: 5432
      POSTGRES_DB: 'ods-notifications'
      POSTGRES_USER: 'notification_usr'
      POSTGRES_PW: 'admin'
      POSTGRES_SSL: 'false'

      CONNECTION_RETRIES: 30
      CONNECTION_BACKOFF_IN_MS: 2000

      AMQP_URL: "amqp://rabbit_adm:R4bb!7_4DM_p4SS@rabbitmq:5672"
      AMQP_PIPELINE_EXECUTION_EXCHANGE: 'ods_global'
      AMQP_PIPELINE_EXECUTION_QUEUE: 'notification.pipeline-execution'
      AMQP_PIPELINE_EXECUTION_SUCCESS_TOPIC: 'pipeline.execution.success'
      AMQP_NOTIFICATION_QUEUE: 'NOTIF_QUEUE'

      NOTIFICATION_DATA_LOCATION_URL: 'http://localhost:9000/storage'
    depends_on:
      - notification-db
      - rabbitmq
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.to-notification.rule=PathPrefix(`/api/notification`)"
      - "traefik.http.routers.to-notification.middlewares=notification-stripprefix@docker"
      - "traefik.http.middlewares.notification-stripprefix.stripprefix.prefixes=/api/notification"

  notification-db:
    image: postgres:13-alpine
    environment:
      POSTGRES_USER: 'notification_usr'
      POSTGRES_PASSWORD: 'admin'
      POSTGRES_DB: 'ods-notifications'

# -------------------- EDGE SERVER (PORT 9000) -----------------------------
  edge:
    image: traefik:2.3
    command:
      - --entrypoints.web.address=:80
      - --providers.docker # enable label annotations in compose file
      - --providers.docker.exposedByDefault=false
      - --api.insecure # Dashboard: don't do that in production
      # - --log.level=DEBUG
    ports:
      - "9000:80"     # The HTTP port
      - "9001:8080"   # The Web UI (enabled by --api)
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock # So that Traefik can listen to the Docker events

# -------------------- RABBIT-MQ -------------------------------------------
  rabbitmq:
    image:  rabbitmq:3-management-alpine
    environment:
      RABBITMQ_ERLANG_COOKIE: 'S0m3_R4bBi7_C0ok13'
    volumes:
      - ./config/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf
    ports:
      - "15672:15672"
      - "5672:5672"