From 1e1d493f8de5ba7da010933318ceda093033b25c Mon Sep 17 00:00:00 2001 From: Scott Black Date: Sun, 7 Jan 2024 12:21:06 -0700 Subject: [PATCH] refactor for use of metadata extractor template ref --- app/argo/huc-to-shp.yml | 22 ++++++++--- app/argo/metadata-extraction.yml | 17 +++++--- app/argo/nwm1-subset-minio.yml | 60 ++++++---------------------- app/argo/nwm2-subset-minio.yml | 64 ++++++------------------------ app/argo/parflow-subset-minio.yml | 66 +++++++------------------------ 5 files changed, 65 insertions(+), 164 deletions(-) diff --git a/app/argo/huc-to-shp.yml b/app/argo/huc-to-shp.yml index ea33b056..fce28074 100644 --- a/app/argo/huc-to-shp.yml +++ b/app/argo/huc-to-shp.yml @@ -1,36 +1,48 @@ metadata: - generateName: huc-to-shp- + name: huc-to-shp namespace: workflows spec: templates: - name: huc-to-shp + inputs: + parameters: + - name: output-bucket + value: '{{workflow.parameters.output-bucket}}' + - name: output-path + value: '{{workflow.parameters.output-path}}' + - name: hucs + value: '{{workflow.parameters.hucs}}' outputs: artifacts: - name: shape-results path: /output + archive: + none: {} s3: endpoint: api.minio.cuahsi.io - bucket: subsetter-outputs + bucket: '{{inputs.parameters.output-bucket}}' accessKeySecret: name: minio-credentials key: accessKey secretKeySecret: name: minio-credentials key: secretKey - key: '{{workflow.parameters.output_path}}/shape.gz' + key: '{{inputs.parameters.output-path}}/shape' metadata: {} container: name: '' image: us-central1-docker.pkg.dev/apps-320517/subsetter/huc-to-shp:0.0.4 args: - - '{{workflow.parameters.hucs}}' + - '{{inputs.parameters.hucs}}' - /output/watershed.shp resources: {} entrypoint: huc-to-shp arguments: parameters: - - name: output_path + - name: output-path value: qwerty + - name: output-bucket + value: subsetter-outputs - name: hucs value: 102600070103,102600070104,102600090205,102600050206,102600070102 templateDefaults: diff --git a/app/argo/metadata-extraction.yml b/app/argo/metadata-extraction.yml index a26dc507..f8e1c058 100644 --- a/app/argo/metadata-extraction.yml +++ b/app/argo/metadata-extraction.yml @@ -8,8 +8,10 @@ spec: parameters: - name: bucket value: '{{workflow.parameters.bucket}}' - - name: path - value: '{{workflow.parameters.path}}' + - name: input-path + value: '{{workflow.parameters.input-path}}' + - name: output-path + value: '{{workflow.parameters.output-path}}' artifacts: - name: input-data path: /tmp/input-data @@ -22,11 +24,13 @@ spec: secretKeySecret: name: minio-credentials key: secretKey - key: '{{inputs.parameters.path}}' + key: '{{inputs.parameters.input-path}}' outputs: artifacts: - name: output-metadata path: /tmp/input-data + archive: + none: {} s3: endpoint: api.minio.cuahsi.io bucket: '{{inputs.parameters.bucket}}' @@ -36,11 +40,11 @@ spec: secretKeySecret: name: minio-credentials key: secretKey - key: '{{inputs.parameters.path}}' + key: '{{inputs.parameters.output-path}}' metadata: {} container: name: '' - image: scootna/hsextract:0.1 + image: scootna/hsextract:0.3 command: - /bin/sh - '-c' @@ -52,7 +56,8 @@ spec: arguments: parameters: - name: bucket - - name: path + - name: input-path + - name: output-path templateDefaults: inputs: {} outputs: {} diff --git a/app/argo/nwm1-subset-minio.yml b/app/argo/nwm1-subset-minio.yml index 1f33e25e..c341a201 100644 --- a/app/argo/nwm1-subset-minio.yml +++ b/app/argo/nwm1-subset-minio.yml @@ -24,14 +24,18 @@ spec: value: '{{workflow.parameters.x_east}}' - name: output-bucket value: '{{workflow.parameters.output-bucket}}' - - - name: metadata-extractor - template: metadata-extractor-template + - - name: metadata-extraction + templateRef: + name: metadata-extractor + template: metadata-extractor arguments: parameters: + - name: bucket + value: '{{workflow.parameters.output-bucket}}' + - name: input-path + value: '{{workflow.parameters.output-path}}' - name: output-path value: '{{workflow.parameters.output-path}}' - - name: output-bucket - value: '{{workflow.parameters.output-bucket}}' - name: nwm1-subset-template inputs: parameters: @@ -58,6 +62,8 @@ spec: artifacts: - name: subset-results path: /srv/output + archive: + none: {} s3: endpoint: api.minio.cuahsi.io bucket: '{{inputs.parameters.output-bucket}}' @@ -67,7 +73,7 @@ spec: secretKeySecret: name: minio-credentials key: secretKey - key: '{{inputs.parameters.output-path}}/subset.gz' + key: '{{inputs.parameters.output-path}}' metadata: {} container: name: '' @@ -84,50 +90,6 @@ spec: - /srv/domain - /srv/output resources: {} - - name: metadata-extractor-template - inputs: - parameters: - - name: output-path - - name: output-bucket - artifacts: - - name: extractor-input - path: /tmp/subset-data - s3: - endpoint: api.minio.cuahsi.io - bucket: '{{inputs.parameters.output-bucket}}' - accessKeySecret: - name: minio-credentials - key: accessKey - secretKeySecret: - name: minio-credentials - key: secretKey - key: '{{inputs.parameters.output-path}}/subset.gz' - outputs: - artifacts: - - name: extractor-output - path: /tmp/subset-data - s3: - endpoint: api.minio.cuahsi.io - bucket: '{{inputs.parameters.output-bucket}}' - accessKeySecret: - name: minio-credentials - key: accessKey - secretKeySecret: - name: minio-credentials - key: secretKey - key: '{{inputs.parameters.output-path}}/all.gz' - metadata: {} - container: - name: '' - image: scootna/hsextract:0.1 - command: - - /bin/sh - - '-c' - args: - - >- - python3 hsextract/main.py extract /tmp/subset-data && mv - /tmp/subset-data/.hs /tmp/subset-data/hs - resources: {} entrypoint: nwm1-subset-with-extraction-template arguments: parameters: diff --git a/app/argo/nwm2-subset-minio.yml b/app/argo/nwm2-subset-minio.yml index 32f08bd6..e25940e1 100644 --- a/app/argo/nwm2-subset-minio.yml +++ b/app/argo/nwm2-subset-minio.yml @@ -24,14 +24,18 @@ spec: value: '{{workflow.parameters.output-path}}' - name: output-bucket value: '{{workflow.parameters.output-bucket}}' - - - name: metadata-extractor - template: metadata-extractor-template + - - name: metadata-extraction + templateRef: + name: metadata-extractor + template: metadata-extractor arguments: parameters: + - name: bucket + value: '{{workflow.parameters.output-bucket}}' + - name: input-path + value: '{{workflow.parameters.output-path}}' - name: output-path value: '{{workflow.parameters.output-path}}' - - name: output-bucket - value: '{{workflow.parameters.output-bucket}}' - name: nwm2-subset-template inputs: parameters: @@ -58,6 +62,8 @@ spec: artifacts: - name: subset-results path: /srv/output + archive: + none: {} s3: endpoint: api.minio.cuahsi.io bucket: '{{inputs.parameters.output-bucket}}' @@ -67,7 +73,7 @@ spec: secretKeySecret: name: minio-credentials key: secretKey - key: '{{inputs.parameters.output-path}}/subset.gz' + key: '{{inputs.parameters.output-path}}' metadata: {} container: name: '' @@ -88,54 +94,6 @@ spec: cpu: '1' ephemeral-storage: 2Gi memory: 1Gi - - name: metadata-extractor-template - inputs: - parameters: - - name: output-path - - name: output-bucket - artifacts: - - name: extractor-input - path: /tmp/subset-data - s3: - endpoint: api.minio.cuahsi.io - bucket: '{{inputs.parameters.output-bucket}}' - accessKeySecret: - name: minio-credentials - key: accessKey - secretKeySecret: - name: minio-credentials - key: secretKey - key: '{{inputs.parameters.output-path}}/subset.gz' - outputs: - artifacts: - - name: extractor-output - path: /tmp/subset-data - s3: - endpoint: api.minio.cuahsi.io - bucket: '{{inputs.parameters.output-bucket}}' - accessKeySecret: - name: minio-credentials - key: accessKey - secretKeySecret: - name: minio-credentials - key: secretKey - key: '{{inputs.parameters.output-path}}/all.gz' - metadata: {} - container: - name: '' - image: scootna/hsextract:0.1 - command: - - /bin/sh - - '-c' - args: - - >- - python3 hsextract/main.py extract /tmp/subset-data && mv - /tmp/subset-data/.hs /tmp/subset-data/hs - resources: - requests: - cpu: '1' - ephemeral-storage: 2Gi - memory: 1Gi entrypoint: nwm2-subset-with-extraction-template arguments: parameters: diff --git a/app/argo/parflow-subset-minio.yml b/app/argo/parflow-subset-minio.yml index b1a0b77b..6898020e 100644 --- a/app/argo/parflow-subset-minio.yml +++ b/app/argo/parflow-subset-minio.yml @@ -28,14 +28,18 @@ spec: value: '{{workflow.parameters.output-path}}' - name: output-bucket value: '{{workflow.parameters.output-bucket}}' - - - name: metadata-extractor - template: metadata-extractor + - - name: metadata-extraction + templateRef: + name: metadata-extractor + template: metadata-extractor arguments: parameters: + - name: bucket + value: '{{workflow.parameters.output-bucket}}' + - name: input-path + value: '{{workflow.parameters.output-path}}' - name: output-path value: '{{workflow.parameters.output-path}}' - - name: output-bucket - value: '{{workflow.parameters.output-bucket}}' - name: huc-to-shp inputs: parameters: @@ -46,6 +50,8 @@ spec: artifacts: - name: shape-results path: /output + archive: + none: {} s3: endpoint: api.minio.cuahsi.io bucket: '{{inputs.parameters.output-bucket}}' @@ -55,7 +61,7 @@ spec: secretKeySecret: name: minio-credentials key: secretKey - key: '{{inputs.parameters.output-path}}/shape.gz' + key: '{{inputs.parameters.output-path}}' metadata: {} container: name: '' @@ -94,11 +100,13 @@ spec: secretKeySecret: name: minio-credentials key: secretKey - key: '{{inputs.parameters.output-path}}/shape.gz' + key: '{{inputs.parameters.output-path}}' outputs: artifacts: - name: subset-results path: /srv/shape + archive: + none: {} s3: endpoint: api.minio.cuahsi.io bucket: '{{inputs.parameters.output-bucket}}' @@ -108,7 +116,7 @@ spec: secretKeySecret: name: minio-credentials key: secretKey - key: '{{inputs.parameters.output-path}}/subset.gz' + key: '{{inputs.parameters.output-path}}' metadata: {} container: name: '' @@ -131,50 +139,6 @@ spec: cpu: '1' ephemeral-storage: 1Gi memory: 1Gi - - name: metadata-extractor - inputs: - parameters: - - name: output-path - - name: output-bucket - artifacts: - - name: extractor-input - path: /tmp/subset-data - s3: - endpoint: api.minio.cuahsi.io - bucket: '{{inputs.parameters.output-bucket}}' - accessKeySecret: - name: minio-credentials - key: accessKey - secretKeySecret: - name: minio-credentials - key: secretKey - key: '{{inputs.parameters.output-path}}/subset.gz' - outputs: - artifacts: - - name: extractor-output - path: /tmp/subset-data - s3: - endpoint: api.minio.cuahsi.io - bucket: '{{inputs.parameters.output-bucket}}' - accessKeySecret: - name: minio-credentials - key: accessKey - secretKeySecret: - name: minio-credentials - key: secretKey - key: '{{inputs.parameters.output-path}}/all.gz' - metadata: {} - container: - name: '' - image: scootna/hsextract:0.1 - command: - - /bin/sh - - '-c' - args: - - >- - python3 hsextract/main.py extract /tmp/subset-data && mv - /tmp/subset-data/.hs /tmp/subset-data/hs - resources: {} entrypoint: parflow-subset-huc arguments: parameters: