Update dockerfile builds, and consolidate broker images. (#3586)

.github/workflows/mkdocs.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout main
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Deploy docs
         uses: mhausenblas/mkdocs-deploy-gh-pages@master

.gitignore

@@ -32,4 +32,5 @@ kindconfig.*
 k3dconfig.*
 lagoon-charts.*
 calico.yaml
-.devcontainer
+.devcontainer
+docker-compose.override.yml

Makefile

@@ -160,7 +160,6 @@ services :=	api \
 			actions-handler \
 			backup-handler \
 			broker \
-			broker-single \
 			keycloak \
 			keycloak-db \
 			logs2notifications \
@@ -185,8 +184,7 @@ build/api-db: services/api-db/Dockerfile
 build/api-redis: services/api-redis/Dockerfile
 build/actions-handler: services/actions-handler/Dockerfile
 build/backup-handler: services/backup-handler/Dockerfile
-build/broker-single: services/broker/Dockerfile
-build/broker: build/broker-single
+build/broker: services/broker/Dockerfile
 build/keycloak-db: services/keycloak-db/Dockerfile
 build/keycloak: services/keycloak/Dockerfile
 build/logs2notifications: services/logs2notifications/Dockerfile
@@ -201,17 +199,11 @@ service-images += ssh
 
 build/local-git: local-dev/git/Dockerfile
 build/local-api-data-watcher-pusher: local-dev/api-data-watcher-pusher/Dockerfile
-build/local-registry: local-dev/registry/Dockerfile
-build/local-dbaas-provider: local-dev/dbaas-provider/Dockerfile
-build/local-mongodb-dbaas-provider: local-dev/mongodb-dbaas-provider/Dockerfile
 build/workflows: services/workflows/Dockerfile
 
 # Images for local helpers that exist in another folder than the service images
 localdevimages := local-git \
-									local-api-data-watcher-pusher \
-									local-registry \
-									local-dbaas-provider \
-									local-mongodb-dbaas-provider
+									local-api-data-watcher-pusher
 
 service-images += $(localdevimages)
 build-localdevimages = $(foreach image,$(localdevimages),build/$(image))
@@ -280,16 +272,6 @@ drupaltest-services-up: main-test-services-up $(foreach image,$(drupal-test-serv
 webhooks-test-services-up: main-test-services-up $(foreach image,$(webhooks-test-services),build/$(image))
 	IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) --compatibility up -d $(webhooks-test-services)
 
-.PHONY: local-registry-up
-local-registry-up: build/local-registry
-	IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) --compatibility up -d local-registry
-
-# broker-up is used to ensure the broker is running before the lagoon-builddeploy operator is installed
-# when running kubernetes tests
-.PHONY: broker-up
-broker-up: build/broker-single
-	IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) --compatibility up -d broker
-
 #######
 ####### Publishing Images
 #######

docker-bake.hcl

@@ -50,15 +50,11 @@ group "default" {
     "api",
     "auth-server",
     "backup-handler",
-    "broker-single",
     "broker",
     "keycloak-db",
     "keycloak",
     "local-api-data-watcher-pusher",
-    "local-dbaas-provider",
     "local-git",
-    "local-mongodb-dbaas-provider",
-    "local-registry",
     "logs2notifications",
     "ssh",
     "task-activestandby",
@@ -75,7 +71,7 @@ group "ui-logs-development" {
     "api-db",
     "api-redis",
     "api",
-    "broker-single",
+    "broker",
     "keycloak-db",
     "keycloak",
     "local-api-data-watcher-pusher",
@@ -86,10 +82,7 @@ group "ui-logs-development" {
 group "local-dev" {
   targets = [
     "local-api-data-watcher-pusher",
-    "local-dbaas-provider",
-    "local-git",
-    "local-mongodb-dbaas-provider",
-    "local-registry",
+    "local-git"
   ]
 }
 
@@ -101,7 +94,6 @@ group "prod-images" {
     "api",
     "auth-server",
     "backup-handler",
-    "broker-single",
     "broker",
     "keycloak-db",
     "keycloak",
@@ -181,21 +173,9 @@ target "backup-handler" {
   tags = ["${IMAGE_REPO}/backup-handler:${TAG}"]
 }
 
-target "broker-single" {
-  inherits = ["default"]
-  context = "services/broker-single"
-  labels = {
-    "org.opencontainers.image.title": "lagoon-core/broker-single - the RabbitMQ broker standalone service for Lagoon"
-  }
-  tags = ["${IMAGE_REPO}/broker-single:${TAG}"]
-}
-
 target "broker" {
   inherits = ["default"]
   context = "services/broker"
-  contexts = {
-    "lagoon/broker-single": "target:broker-single"
-  }
   labels = {
     "org.opencontainers.image.title": "lagoon-core/broker - the RabbitMQ broker service for Lagoon"
   }
@@ -301,15 +281,6 @@ target "local-api-data-watcher-pusher" {
   tags = ["${IMAGE_REPO}/local-api-data-watcher-pusher:${TAG}"]
 }
 
-target "local-dbaas-provider" {
-  inherits = ["default"]
-  context = "local-dev/dbaas-provider"
-  labels = {
-    "org.opencontainers.image.title": "lagoon-core/local-dbaas-provider - the local-dev MariaDB DBaaS image for Lagoon"
-  }
-  tags = ["${IMAGE_REPO}/local-dbaas-provider:${TAG}"]
-}
-
 target "local-git" {
   inherits = ["default"]
   context = "local-dev/git"
@@ -318,21 +289,3 @@ target "local-git" {
   }
   tags = ["${IMAGE_REPO}/local-git:${TAG}"]
 }
-
-target "local-mongodb-dbaas-provider" {
-  inherits = ["default"]
-  context = "local-dev/mongodb-dbaas-provider"
-  labels = {
-    "org.opencontainers.image.title": "lagoon-core/local-mongodb-dbaas-provider - the local-dev MongoDB DBaaS image for Lagoon"
-  }
-  tags = ["${IMAGE_REPO}/local-mongodb-dbaas-provider:${TAG}"]
-}
-
-target "local-registry" {
-  inherits = ["default"]
-  context = "local-dev/registry"
-  labels = {
-    "org.opencontainers.image.title": "lagoon-core/local-registry - the local-dev Docker registry image for Lagoon"
-  }
-  tags = ["${IMAGE_REPO}/local-registry:${TAG}"]
-}

docker-compose.yaml

@@ -22,7 +22,7 @@ services:
     depends_on:
       - broker
   broker:
-    image: ${IMAGE_REPO:-lagoon}/broker-single
+    image: ${IMAGE_REPO:-lagoon}/broker
     ports:
       - '15672:15672'
       - '5672:5672'
@@ -70,7 +70,7 @@ services:
       - S3_FILES_HOST=http://172.17.0.1:9000
       - S3_BAAS_ACCESS_KEY_ID=minio
       - S3_BAAS_SECRET_ACCESS_KEY=minio123
-      - CONSOLE_LOGGING_LEVEL=trace
+      - CONSOLE_LOGGING_LEVEL=debug
     depends_on:
       - api-init
       - keycloak
@@ -167,35 +167,6 @@ services:
       - DELETED_STATUS_CODE=404
     volumes:
       - ./tests:/ansible
-  tests-openshift:
-    image: ${IMAGE_REPO:-lagoon}/tests
-    environment:
-      - CLUSTER_TYPE=openshift
-      - WEBHOOK_HOST=webhook-handler
-      - WEBHOOK_PORT=3000
-      - WEBHOOK_PROTOCOL=http
-      - ROUTE_SUFFIX_HTTP=192.168.42.103.nip.io
-      - ROUTE_SUFFIX_HTTP_PORT=80
-      - ROUTE_SUFFIX_HTTPS=192.168.42.103.nip.io
-      - ROUTE_SUFFIX_HTTPS_PORT=443
-      - WEBHOOK_REPO_PREFIX=ssh://[email protected]:2222/git/
-      - GIT_REPO_PREFIX=git@local-git:/git/
-      - SSH_PRIVATE_KEY=-----BEGIN RSA PRIVATE KEY-----\nMIIJKAIBAAKCAgEAxGZZrOV7Islo5p51Moabfd1YB8qbHvQZfJDZJmSU4jNxMf8G\nQH8KIM6ndi60xIiDlkh9R50Gs0fgnPaBwpjMjcUay5EvzBJdMmd/EPhg359+4f5Z\nQiGTVaB5UoGJKg9DEK4Xxi+QLpQ1CiJXvd3QOqnQlJZ2WYwz4kdLxF0d+sRrl+0G\nAISt9Gg9kq6wa7k7RRr4/OyD/9DhDr1GXvHXFPRv6QqKq084CqiUaarP7OcbZKi5\nEyMkf0s86ZTttQPqQijWsenLAw6t7J0vM38eojPDMFX4fJ7wVvbHmsdcwb2YxwD0\nk7I85mV5uM99v7owJ/0YSJGcN2YESq8c68rbRp1iSfDhchd9SUyYMFlrj3R+dxte\nTwvN2W5hIEld8Ip+dUWUVaaTeLkFLmqmVhvcMJNmuj+Wnp9USwki6U5HdpgBJPT5\nYJia3/LyE5IlPaRfZ+vBDQqKOeeqkncVtqcZ468ylT0qpqjtV4OSuCzl+P/TeJ+K\npCxDoqp88yQpYRYn9ztvEVvOkT8RERnT0/NVCNKAwHFOXrkK/BJs/h3fj2NddeVC\nJXdwiB4m+T2C/RHtGxVColqUf2nEntXxweuJgqBz+4YxXqRrKu4p5L4NuudcFAyg\nbIRpbaTZDI+vmmXnTXNP6ymMo5fNJ0/IPEBAoDkkc6ZmKdM5Yth6RaCzolsCAwEA\nAQKCAgBRL4n0SaxcFevyISCLMU+HeP8RwXPcGMWMU4ggMcXFt8QcCbK46Ir0QtjA\nps/8yf2zGuYGu2dwps63s8KyAV3VYNwRuEOM1S6HTncdOb850YGuw+h7yMtRwxND\nwt5Db6mmdIDJYRH13zgJp2ajytRv25CoS4ZCwA0WhahRVLi214kudUFc53vNI2YI\ng4PUE+7nQx4X12E9V0ghQqabStdBB0ZXjA8Ef6vH5CXthDmNUX9mXcSbn5RPtndI\ni1Kz2Bl3HdCaHO3ZprDItbU0UWEFZeZSzcb5JO5u1HZwiebTA5/q638uqqg4CUyG\n0+bEYZ/Ud/AY13A/CkHN6ZNH+UQotCH3GLyaTQq6OhyXlgMBojn3xs9iMUclFcuy\nkbZ1jAxqci25pxCIeNDHBDKRyxgSkDPna8ZZ4eKGXjIZzsds4+IDkYJLMg0OCtd2\nKm+ZPM2CFU2YCqt11dlr0higGK/9gdpajJMVgEYAmigQ670LdcBc4XIOpPMrR70a\nPjbF9ze/UqtKDWtz8KMIUcvr0CjhHa3XRgMJiM34hIOZU6xf6rjEgkN2Geq24u1b\nhUW8URb+5dcasQ9iIfkNn3R+zK5PzyvQNf6+XrbVjpLqPHXJYrD85EKlXkPqiE6y\n3ehYMrIaZIY6gMuIVcg8AEtsm5EwQY7ETw4YnMQLKfyweLsHEQKCAQEA5vavDyQj\nn6PqfF1Ntr3N/FfwRr/167HB+/3wHT+FwRpCw62ACeBC0iFXmzG2wfQfot60fUGa\nQEJiUDGZFvBM0tiTpzmgGG7QpRbPe1919Sl5LZeLA9bngRnmqn5zAkmVTeBCO/4F\nMSk9hnBZ0v0W3DqNmjuIH298g3gl4VJeKpILd62LbhjvhjT/LXlekYDEj3p9Xob8\n1OQxFJgftFyg4P07RRaUzsNLhqEdY3VxDcUMb9459fEYeb7sYig+zVPaZQ31aMVK\nj6K+XiH5M5uKJWkPdMDUG84jreFAdBY9kGCuSulTWgmTLlMKSI85q5hkckY2EQQL\n5u456xfyuFcnEwKCAQEA2bCCcqcGIAiMwk/6z7XIJeUsSUg+ObvkEdogk5n6Y1Ea\nt5LxMpQzhG6SHJ2f38VyKgv9e/jnwXI8eiejper6OeQEBG4+AedcLYi0V5SUMIgD\nX4bxT9+qCwYrwt9YHkJySk237WZUWJPVfxHg0vqNYyD/CXBowx0nm8jEuZ8iT+CW\nO2uZq+0DO2WqoYT54lZux6aEzm+oAkzwJJVXJcUVPg7bJXK1ObOzvHpkZJxHL8+S\nKufzew/CXALKWHoCkEP/P8b7oxjcjQI3KK0EM2fABNvN28+qscqTqQjfAsNw24Ob\nP8rL8amgd/x7iddIbEpOSoLAH1gVoxJXA0oqkC6YmQKCAQEAiIeoomW1nUgTdCLf\nrrfyzUnaoqgVIpf42RKa319OnQD+GJg2iSAFwBuvovE3XN4H2YqW3Thri7LyVP+M\nxM+WSkl2tzLMXcUcF4staXvbyeoTVQ0xQzrFrT53aa/IIsEGvntkC/y0awQ6937w\nylWMLvF6BYNNi2+nBjx+//xl5/pfRwbS1mltJkOr2ODXM2IQT9STyc44JU0jak4m\n58Kd44IuiD+6BaPSwKn7KnEhPIeQO4z9bFJyKn3fVIL/5Pa9smEXAjEmS1Rj/ldM\n7eHzPvwlA9p9SFaKJt5x8G25ROCyB1x4RlBEreyutofcCoDSV+8DRPnEY2XN3RhS\nBgCW+wKCAQAyHrqaDSyS2YUXA/UaN59CYlZk9PYmssTa+16+vRfyH+1H0NQpsgd+\neTq4ze3ORpZ3adVN4svxNQ0JjvDVtZkxFPd70aoDJDKL5UqoU3QfDGHCb75FhpiO\n+ze+IVAXf3Ly+pvbU9Edp8PjAsnBqaA9743orXHhYmgJLRtmJWZv/6R3P9ncxLAW\nz9yOXaBu5TmSTBuNsBV9mhs8QQoUjyDJ7f+0yolBJMfAIGnW5EhbkK31pPvhHIrC\nRn4yCr1mW9F77KNpNMMq0BTFD7jE4SFLvRPThSem0Z5Xr8wwxbDJSa7H7DgyhryE\ng6Qp42AwVpdZ/mqfjNdGeWWBQ2UzVxcZAoIBAHNXgjD3umcKciYRAbltNJrXx9xk\ndAv8I69oEMCy4hCmvKUjy/UI9NqXFjBb/G6VGgh6NUE9o9o27t1Y5Plm0izyCA1M\nDFruaRfjyJ8qjbEifcqRtcF3rzsBiXIwdmbN6qT4PUipN2elpUAd7J1OIwGIIe3u\nCWNyOTNL+2+oQ/Eg1Y99mg3yrsVyOwhynVE80/X5cy07bXXR5rv1x4NKSVbPhlnt\nL6J5iAoqoDKICzjcgF5x3mj9YFWZrC3aRxRrN5RoEgeVdcXeK56UJqXHjmKN++m3\nc8OPEIBZiD8UJuhSNSOLiBFrGz6toy6rpHavqqknGhVWotXsAs1h8LNkBe8=\n-----END RSA PRIVATE KEY-----
-      - SSH_HOST=ssh
-      - SSH_PORT=2020
-      - API_HOST=api
-      - API_PROTOCOL=http
-      - API_PORT=3000
-      - KEYCLOAK_AUTH_SERVER_CLIENT_SECRET=f605b150-7636-4447-abd3-70988786b330
-      - DELETED_STATUS_CODE=503
-    volumes:
-      - ./tests:/ansible
-  local-git:
-    image: ${IMAGE_REPO:-lagoon}/local-git
-    environment:
-      - GIT_AUTHORIZED_KEYS=ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDEZlms5XsiyWjmnnUyhpt93VgHypse9Bl8kNkmZJTiM3Ex/wZAfwogzqd2LrTEiIOWSH1HnQazR+Cc9oHCmMyNxRrLkS/MEl0yZ38Q+GDfn37h/llCIZNVoHlSgYkqD0MQrhfGL5AulDUKIle93dA6qdCUlnZZjDPiR0vEXR36xGuX7QYAhK30aD2SrrBruTtFGvj87IP/0OEOvUZe8dcU9G/pCoqrTzgKqJRpqs/s5xtkqLkTIyR/SzzplO21A+pCKNax6csDDq3snS8zfx6iM8MwVfh8nvBW9seax1zBvZjHAPSTsjzmZXm4z32/ujAn/RhIkZw3ZgRKrxzryttGnWJJ8OFyF31JTJgwWWuPdH53G15PC83ZbmEgSV3win51RZRVppN4uQUuaqZWG9wwk2a6P5aen1RLCSLpTkd2mAEk9PlgmJrf8vITkiU9pF9n68ENCoo556qSdxW2pxnjrzKVPSqmqO1Xg5K4LOX4/9N4n4qkLEOiqnzzJClhFif3O28RW86RPxERGdPT81UI0oDAcU5euQr8Emz+Hd+PY1115UIld3CIHib5PYL9Ee0bFUKiWpR/acSe1fHB64mCoHP7hjFepGsq7inkvg2651wUDKBshGltpNkMj6+aZedNc0/rKYyjl80nT8g8QECgOSRzpmYp0zli2HpFoLOiWw== ansible-testing
-    ports:
-      - '2222:22'
   remotedev:
     image: jhen0409/remotedev-server
     platform: linux/amd64
@@ -213,21 +184,6 @@ services:
       - CLEAR_API_DATA=false
     volumes:
       - ./local-dev/api-data-watcher-pusher:/home
-  local-dbaas-provider:
-    image: ${IMAGE_REPO:-lagoon}/local-dbaas-provider
-    restart: always
-    environment:
-      MYSQL_DATABASE: 'db'
-      MYSQL_USER: 'user'
-      MYSQL_PASSWORD: 'password'
-      MYSQL_ROOT_PASSWORD: 'password'
-    ports:
-      - '3306:3306'
-  local-mongodb-dbaas-provider:
-    image: ${IMAGE_REPO:-lagoon}/local-mongodb-dbaas-provider
-    restart: always
-    ports:
-      - '27017:27017'
   local-minio:
     image: minio/minio
     entrypoint: sh

local-dev/api-data-watcher-pusher/Dockerfile

@@ -7,7 +7,6 @@ RUN apk add --no-cache \
       minio-client \
       moreutils \
       py3-jwt \
-      tini \
       wget
 
 ENV JWTSECRET=super-secret-string \
@@ -22,4 +21,4 @@ COPY api-data /home/api-data
 COPY minio-data /home/minio-data
 COPY data-init-push.sh create_jwt.py /home/
 
-CMD ["/sbin/tini", "--", "/home/data-init-push.sh"]
+CMD ["/sbin/tini", "--", "/home/data-init-push.sh"]

local-dev/git/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.17.1
+FROM ${UPSTREAM_REPO:-uselagoon}/commons:${UPSTREAM_TAG:-latest}
 
 RUN apk add --no-cache \
       bash \
@@ -50,4 +50,4 @@ USER root
 
 COPY entrypoint.sh /usr/sbin/entrypoint.sh
 
-ENTRYPOINT ["/usr/sbin/entrypoint.sh"]
+ENTRYPOINT ["/sbin/tini", "--", "/usr/sbin/entrypoint.sh"]