Merge pull request #124 from overture-stack/rc/0.15.0

Rc/0.15.0
add SSL configuration in a secure nginx image
upgrade dms-ui tag
upgrade arranger-ui tag

Jenkinsfile.groovy

@@ -111,7 +111,8 @@ spec:
                     }
                     sh "docker build --network=host --target client -f  Dockerfile . -t ${dockerOrg}/${dmsRepo}:edge -t ${dockerOrg}/${dmsRepo}:${version}-${commit}"
                     sh "docker build --network=host --target latest-version-helper -f Dockerfile . -t ${dockerOrg}/${dmsVersionHelperRepo}:edge"
-                    sh "docker build --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}:edge"
+                    sh "docker build --target insecure --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}:edge"
+                    sh "docker build --target secure --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}-secure:edge"
                     sh "docker push ${dockerOrg}/${dmsRepo}:${version}-${commit}"
                     sh "docker push ${dockerOrg}/${dmsRepo}:edge"
                     sh "docker push ${dockerOrg}/${dmsGatewayRepo}:edge"
@@ -134,10 +135,13 @@ spec:
                     }
                     sh "docker build --network=host --target client -f Dockerfile . -t ${dockerOrg}/${dmsRepo}:latest -t ${dockerOrg}/${dmsRepo}:${version}"
                     sh "docker build --network=host --target latest-version-helper  -f Dockerfile . -t ${dockerOrg}/${dmsVersionHelperRepo}:latest"
-                    sh "docker build --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}:latest -t ${dockerOrg}/${dmsGatewayRepo}:${version}"
+                    sh "docker build --target insecure --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}:latest -t ${dockerOrg}/${dmsGatewayRepo}:${version}"
+                    sh "docker build --target secure --network=host -f ./nginx/path-based/Dockerfile ./nginx/path-based -t ${dockerOrg}/${dmsGatewayRepo}-secure:latest -t ${dockerOrg}/${dmsGatewayRepo}-secure:${version}"
                     sh "docker push ${dockerOrg}/${dmsRepo}:${version}"
                     sh "docker push ${dockerOrg}/${dmsGatewayRepo}:${version}"
+                    sh "docker push ${dockerOrg}/${dmsGatewayRepo}-secure:${version}"
                     sh "docker push ${dockerOrg}/${dmsGatewayRepo}:latest"
+                    sh "docker push ${dockerOrg}/${dmsGatewayRepo}-secure:latest"
                     sh "docker push ${dockerOrg}/${dmsRepo}:latest"
                     sh "docker push ${dockerOrg}/${dmsVersionHelperRepo}:latest"
                 }

Makefile

@@ -78,8 +78,8 @@ build-image:
 	@$(DOCKER_EXE) build -t $(DOCKER_IMAGE_NAME) ./
 
 build-gateway:
-	docker build ./nginx/path-based -t $(DOCKER_ORG)/$(DOCKER_GATEWAY_REPO):edge --no-cache -f./nginx/path-based/Dockerfile
-
+	docker build --target insecure ./nginx/path-based -t $(DOCKER_ORG)/$(DOCKER_GATEWAY_REPO):edge --no-cache -f./nginx/path-based/Dockerfile
+	docker build --target secure ./nginx/path-based -t $(DOCKER_ORG)/$(DOCKER_GATEWAY_REPO)-secure:edge --no-cache -f./nginx/path-based/Dockerfile
 push-image: build-image
 	@$(DOCKER_EXE) push $(DOCKER_IMAGE_NAME)
 

README.md

@@ -41,3 +41,6 @@ tagging the gateway is done in Jenkinsfile, it will always have a new tag with t
 Sometimes, if the reserved/limit memory is too low, a container will get killed by the kernel. To find out if this is the case, run
 `journalctl -k | grep -i -e memory -e oom`. For java apps, the status `"task: non-zero exit (137)"` is usually the case.
 
+## useful commands:
+- `docker service ps --no-trunc {serviceName}`
+- `journalctl -u docker.service | tail -n 50 `

nginx/path-based/Dockerfile

@@ -1,2 +1,6 @@
-FROM nginx:1.19.7
-COPY ./default.conf.template /etc/nginx/templates/default.conf.template
+FROM nginx:1.19.7 as insecure
+COPY ./default.conf.template /etc/nginx/templates/default.conf.template
+
+
+FROM nginx:1.19.7 as secure
+COPY ./default.secure.conf.template /etc/nginx/templates/default.conf.template

nginx/path-based/default.secure.conf.template

@@ -0,0 +1,150 @@
+server {
+    listen 80;
+    server_name ${NGINX_HOST};
+    return 301 https://$server_name$request_uri;
+}
+
+server {
+    listen       443 ssl;
+    server_name  ${NGINX_HOST};
+    # https://serverspace.us/support/help/ssl-certificate-on-nginx-ubuntu-20-04/
+    # these paths are based on mounting /etc/letsencrypt/ assuming the use of certbot, although
+    # this would work with any cert in the same dir structure
+    # this is needed this way to support symlinks that certbot needs to auto renew
+    ssl_certificate /dms/ssl/live/${NGINX_HOST}/fullchain.pem;
+    ssl_certificate_key /dms/ssl/live/${NGINX_HOST}/privkey.pem;
+
+    # https://www.techrepublic.com/article/how-to-enable-ssl-on-nginx/
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_prefer_server_ciphers on;
+    #ssl_dhparam /dms/ssl/dhparam.pem;
+    ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384;
+    ssl_ecdh_curve secp384r1; # Requires nginx >= 1.1.0
+    ssl_session_timeout  10m;
+    ssl_session_cache shared:SSL:10m;
+    ssl_session_tickets off; # Requires nginx >= 1.5.9
+    # ssl_stapling on; # Requires nginx >= 1.3.7
+    # ssl_stapling_verify on; # Requires nginx => 1.3.7
+    #resolver 8.8.8.8 8.8.4.4 valid=300s;
+    #resolver_timeout 5s;
+
+    add_header X-Frame-Options DENY;
+    add_header X-Content-Type-Options nosniff;
+    add_header X-XSS-Protection "1; mode=block";
+
+    # To allow special characters in headers
+    ignore_invalid_headers off;
+
+    # Allow any size file to be uploaded.
+    # Set to a value such as 1000m; to restrict file size to a specific value
+    client_max_body_size 0;
+
+    # To disable buffering
+    proxy_buffering off;
+
+    resolver 127.0.0.11 valid=1s;
+
+    set $ego http://ego-api:8080;
+    set $minio http://minio-api:9000;
+    set $song http://song-api:8080;
+    set $score http://score-api:8080;
+    set $es http://elasticsearch:9200;
+    set $maestro http://maestro:11235;
+    set $arranger_ui http://arranger-ui:8080;
+    set $arranger_server http://arranger-server:5050;
+    set $dms_ui http://dms-ui:3000;
+    set $egoui http://ego-ui:8080;
+
+    # this is for the minio ui browser
+    # won't work if you change it to something else it's reserved
+    # by minio to be /minio
+    location /minio {
+        # break is important for score upload
+        proxy_set_header Host minio-api:9000;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_connect_timeout 300;
+
+        # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
+        chunked_transfer_encoding off;
+        proxy_pass $minio;
+        # important for score upload.
+        proxy_redirect  off;
+    }
+
+    # configs from https://docs.minio.io/docs/setup-nginx-proxy-with-minio
+    location /minio-api {
+        # break is important for score upload
+        rewrite /minio-api/(.*) /$1 break;
+        proxy_set_header Host minio-api:9000;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_connect_timeout 300;
+
+        # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
+        chunked_transfer_encoding off;
+        proxy_pass $minio;
+        # important for score upload.
+        proxy_redirect  off;
+    }
+
+    location /ego-api {
+        rewrite /ego-api/(.*) /$1  break;
+        proxy_pass $ego;
+        proxy_redirect off;
+    }
+
+    location /ego-ui {
+        rewrite /ego-ui(.*) /$1  break;
+        proxy_pass $egoui;
+        proxy_redirect  off;
+    }
+
+    location /song-api {
+        rewrite /song-api/(.*) /$1  break;
+        proxy_pass $song;
+        proxy_redirect  off;
+    }
+
+    location /score-api {
+        rewrite /score-api/(.*) /$1  break;
+        proxy_pass  $score;
+        proxy_redirect  off;
+    }
+
+    location /elasticsearch {
+        rewrite /elasticsearch/(.*) /$1  break;
+        proxy_pass $es;
+        proxy_redirect  off;
+    }
+
+    location /maestro {
+        rewrite /maestro/(.*) /$1  break;
+        proxy_pass $maestro;
+        proxy_redirect  off;
+    }
+
+    location /arranger-ui {
+        rewrite /arranger-ui/(.*) /$1  break;
+        proxy_pass $arranger_ui;
+        proxy_redirect  off;
+    }
+
+    location /arranger-server {
+        rewrite /arranger-server/(.*) /$1  break;
+        proxy_pass $arranger_server;
+        proxy_redirect  off;
+    }
+
+    location /dms-ui {
+        rewrite /dms-ui/(.*) /$1  break;
+        proxy_pass $dms_ui;
+        proxy_redirect  off;
+    }
+}

pom.xml

@@ -10,7 +10,7 @@
 	</parent>
 	<groupId>bio.overture</groupId>
 	<artifactId>dms</artifactId>
-  	<version>0.14.0</version>
+  	<version>0.15.0</version>
 	<name>dms</name>
 	<description>Overture Data Management System</description>
 

src/main/java/bio/overture/dms/cli/questionnaire/DmsQuestionnaire.java

@@ -1,6 +1,5 @@
 package bio.overture.dms.cli.questionnaire;
 
-import static bio.overture.dms.core.model.enums.ClusterRunModes.LOCAL;
 import static bio.overture.dms.core.model.enums.ClusterRunModes.SERVER;
 
 import bio.overture.dms.cli.question.QuestionFactory;
@@ -73,16 +72,31 @@ public DmsConfig buildDmsConfig() {
                 ClusterRunModes.class, "Select the cluster mode to configure: ", false, null)
             .getAnswer();
 
-    GatewayConfig gatewayConfig = null;
+    GatewayConfig gatewayConfig;
     URL dmsGatewayUrl;
-    int gatewayPort = 80;
+    int gatewayPort;
+    String sslPath = "/etc/ssl/dms";
     if (clusterRunMode == SERVER) {
       dmsGatewayUrl =
           questionFactory
-              .newUrlSingleQuestion("What is the base DMS Gateway URL (example: http://dms.cancercollaboratory.org)?",
+              .newUrlSingleQuestion("What is the base DMS Gateway URL (example: https://dms.cancercollaboratory.org)?",
                   false,
                   null
               ).getAnswer();
+
+      if (dmsGatewayUrl.getPort() <= 0) {
+        dmsGatewayUrl = new URI("https", null, dmsGatewayUrl.getHost(),
+            443, null, null, null).toURL();
+      }
+
+       sslPath =
+          questionFactory
+              .newDefaultSingleQuestion(String.class,"What is the absolute path for the SSL certificate ?",
+                  false,
+                  "/etc/letsencrypt/"
+              ).getAnswer();
+
+      gatewayPort = 443;
     } else {
       gatewayPort =
           questionFactory
@@ -92,12 +106,16 @@ public DmsConfig buildDmsConfig() {
                   true,
                   80
               ).getAnswer();
-      dmsGatewayUrl = new URI("http", null, "localhost", gatewayPort, null, null, null).toURL();
+      dmsGatewayUrl = new URI("http", null, "localhost",
+          gatewayPort, null, null, null).toURL();
     }
+
     gatewayConfig = GatewayConfig.builder()
         .hostPort(gatewayPort)
         .url(dmsGatewayUrl)
+        .sslDir(sslPath)
         .build();
+
     printHeader("EGO");
     val egoConfig = egoQuestionnaire.buildEgoConfig(clusterRunMode, gatewayConfig);
     printHeader("SONG");

src/main/java/bio/overture/dms/compose/deployment/DmsComposeManager.java

@@ -75,12 +75,8 @@ public void deploy(@NonNull DmsConfig dmsConfig) {
     swarmService.getOrCreateNetwork(dmsConfig.getNetwork());
     val completableFutures = new ArrayList<CompletableFuture<?>>();
     CompletableFuture<Void> gateway;
-    if (dmsConfig.getGateway().isPathBased() || dmsConfig.getClusterRunMode() == ClusterRunModes.SERVER) {
-      gateway = runAsync(getDeployRunnable(dmsConfig, GATEWAY, messenger), executorService);
-      completableFutures.add(gateway);
-    } else {
-      gateway = CompletableFuture.completedFuture(null);
-    }
+    gateway = runAsync(getDeployRunnable(dmsConfig, GATEWAY, messenger), executorService);
+    completableFutures.add(gateway);
 
     val egoFuture =
         gateway.thenRunAsync(() -> egoApiDbDeployer.deploy(dmsConfig), executorService)
@@ -111,7 +107,7 @@ public void deploy(@NonNull DmsConfig dmsConfig) {
     completableFutures.add(scoreApiFuture);
 
     val elasticMaestroFuture =
-        runAsync(() -> elasticsearchDeployer.deploy(dmsRunningInDocker, dmsConfig), executorService)
+        gateway.thenRunAsync(() -> elasticsearchDeployer.deploy(dmsRunningInDocker, dmsConfig), executorService)
             .thenRunAsync(
                 getMaestroDeployRunnable(dmsConfig, dmsRunningInDocker, messenger),
                 executorService);

src/main/java/bio/overture/dms/core/model/dmsconfig/DmsConfig.java

@@ -18,8 +18,8 @@
 @NoArgsConstructor(force = true, access = PRIVATE)
 @JsonInclude(NON_EMPTY)
 public class DmsConfig {
-
   @NonNull private final GatewayConfig gateway;
+
   @NonNull private final HealthCheckConfig healthCheck;
 
   @NonNull private final ClusterRunModes clusterRunMode;

src/main/java/bio/overture/dms/core/model/dmsconfig/GatewayConfig.java

@@ -23,4 +23,5 @@ public class GatewayConfig {
   @Builder.Default
   private int hostPort = 80;
   @NotNull private URL url;
+  private String sslDir = "/etc/ssl/dms";
 }

src/main/resources/templates/servicespec/arranger-ui.yaml.vm

@@ -5,7 +5,7 @@
 Name: arranger-ui
 TaskTemplate:
   ContainerSpec:
-    Image: "overture/arranger-ui:2.11.1-056c5ed"
+    Image: "overture/arranger-ui:2.11.1-bb38e27"
     Env:
       - PORT=8080
       # no slash at beginning is intended otherwise arranger won't work correctly

src/main/resources/templates/servicespec/dms-ui.yaml.vm

@@ -5,7 +5,7 @@
 Name: dms-ui
 TaskTemplate:
   ContainerSpec:
-    Image: "overture/dms-ui:0.6.0-f08019f"
+    Image: "overture/dms-ui:0.7.0"
     Env:
       - NEXT_PUBLIC_EGO_API_ROOT=$dmsConfig.ego.api.url
       - NEXT_PUBLIC_EGO_CLIENT_ID=dms

src/main/resources/templates/servicespec/gateway.yaml.vm

@@ -5,11 +5,21 @@
 Name: gateway
 TaskTemplate:
   ContainerSpec:
-    Image: "ghcr.io/overture-stack/dms-gateway:0.13.0"
+#if( $dmsConfig.clusterRunMode == 'SERVER')
+    Image: "ghcr.io/overture-stack/dms-gateway-secure:0.15.0"
+#else
+    Image: "ghcr.io/overture-stack/dms-gateway:0.15.0"
+#end
     Env:
-      - NGINX_PORT=80
-      - NGINX_HOST=localhost
-    Mounts: null
+      - NGINX_PORT=$dmsConfig.ego.api.url.port
+      - NGINX_HOST=$dmsConfig.gateway.url.host
+#if( $dmsConfig.clusterRunMode == 'SERVER')
+    Mounts:
+      - Type: bind
+        Source: $dmsConfig.gateway.sslDir
+        Target: /dms/ssl
+        ReadOnly: true
+#end
     Duration: null
     StopGracePeriod: 120000000000
     DNSConfig: null
@@ -66,10 +76,23 @@ Networks:
 EndpointSpec:
   Mode: "vip"
   Ports:
+#if ($dmsConfig.clusterRunMode == 'LOCAL')
     - Name: "http"
       Protocol: "tcp"
       TargetPort: 80
       PublishedPort: $dmsConfig.gateway.hostPort
       PublishMode: "ingress"
+#else
+    - Name: "http"
+      Protocol: "tcp"
+      TargetPort: 80
+      PublishedPort: 80
+      PublishMode: "ingress"
+    - Name: "https"
+      Protocol: "tcp"
+      TargetPort: 443
+      PublishedPort: 443
+      PublishMode: "ingress"
+#end
 Labels: null