Skip to content

Commit

Permalink
refactor for use of metadata extractor template ref
Browse files Browse the repository at this point in the history
  • Loading branch information
sblack-usu committed Jan 7, 2024
1 parent 8943683 commit 1e1d493
Show file tree
Hide file tree
Showing 5 changed files with 65 additions and 164 deletions.
22 changes: 17 additions & 5 deletions app/argo/huc-to-shp.yml
Original file line number Diff line number Diff line change
@@ -1,36 +1,48 @@
metadata:
generateName: huc-to-shp-
name: huc-to-shp
namespace: workflows
spec:
templates:
- name: huc-to-shp
inputs:
parameters:
- name: output-bucket
value: '{{workflow.parameters.output-bucket}}'
- name: output-path
value: '{{workflow.parameters.output-path}}'
- name: hucs
value: '{{workflow.parameters.hucs}}'
outputs:
artifacts:
- name: shape-results
path: /output
archive:
none: {}
s3:
endpoint: api.minio.cuahsi.io
bucket: subsetter-outputs
bucket: '{{inputs.parameters.output-bucket}}'
accessKeySecret:
name: minio-credentials
key: accessKey
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{workflow.parameters.output_path}}/shape.gz'
key: '{{inputs.parameters.output-path}}/shape'
metadata: {}
container:
name: ''
image: us-central1-docker.pkg.dev/apps-320517/subsetter/huc-to-shp:0.0.4
args:
- '{{workflow.parameters.hucs}}'
- '{{inputs.parameters.hucs}}'
- /output/watershed.shp
resources: {}
entrypoint: huc-to-shp
arguments:
parameters:
- name: output_path
- name: output-path
value: qwerty
- name: output-bucket
value: subsetter-outputs
- name: hucs
value: 102600070103,102600070104,102600090205,102600050206,102600070102
templateDefaults:
Expand Down
17 changes: 11 additions & 6 deletions app/argo/metadata-extraction.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ spec:
parameters:
- name: bucket
value: '{{workflow.parameters.bucket}}'
- name: path
value: '{{workflow.parameters.path}}'
- name: input-path
value: '{{workflow.parameters.input-path}}'
- name: output-path
value: '{{workflow.parameters.output-path}}'
artifacts:
- name: input-data
path: /tmp/input-data
Expand All @@ -22,11 +24,13 @@ spec:
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{inputs.parameters.path}}'
key: '{{inputs.parameters.input-path}}'
outputs:
artifacts:
- name: output-metadata
path: /tmp/input-data
archive:
none: {}
s3:
endpoint: api.minio.cuahsi.io
bucket: '{{inputs.parameters.bucket}}'
Expand All @@ -36,11 +40,11 @@ spec:
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{inputs.parameters.path}}'
key: '{{inputs.parameters.output-path}}'
metadata: {}
container:
name: ''
image: scootna/hsextract:0.1
image: scootna/hsextract:0.3
command:
- /bin/sh
- '-c'
Expand All @@ -52,7 +56,8 @@ spec:
arguments:
parameters:
- name: bucket
- name: path
- name: input-path
- name: output-path
templateDefaults:
inputs: {}
outputs: {}
Expand Down
60 changes: 11 additions & 49 deletions app/argo/nwm1-subset-minio.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,18 @@ spec:
value: '{{workflow.parameters.x_east}}'
- name: output-bucket
value: '{{workflow.parameters.output-bucket}}'
- - name: metadata-extractor
template: metadata-extractor-template
- - name: metadata-extraction
templateRef:
name: metadata-extractor
template: metadata-extractor
arguments:
parameters:
- name: bucket
value: '{{workflow.parameters.output-bucket}}'
- name: input-path
value: '{{workflow.parameters.output-path}}'
- name: output-path
value: '{{workflow.parameters.output-path}}'
- name: output-bucket
value: '{{workflow.parameters.output-bucket}}'
- name: nwm1-subset-template
inputs:
parameters:
Expand All @@ -58,6 +62,8 @@ spec:
artifacts:
- name: subset-results
path: /srv/output
archive:
none: {}
s3:
endpoint: api.minio.cuahsi.io
bucket: '{{inputs.parameters.output-bucket}}'
Expand All @@ -67,7 +73,7 @@ spec:
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{inputs.parameters.output-path}}/subset.gz'
key: '{{inputs.parameters.output-path}}'
metadata: {}
container:
name: ''
Expand All @@ -84,50 +90,6 @@ spec:
- /srv/domain
- /srv/output
resources: {}
- name: metadata-extractor-template
inputs:
parameters:
- name: output-path
- name: output-bucket
artifacts:
- name: extractor-input
path: /tmp/subset-data
s3:
endpoint: api.minio.cuahsi.io
bucket: '{{inputs.parameters.output-bucket}}'
accessKeySecret:
name: minio-credentials
key: accessKey
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{inputs.parameters.output-path}}/subset.gz'
outputs:
artifacts:
- name: extractor-output
path: /tmp/subset-data
s3:
endpoint: api.minio.cuahsi.io
bucket: '{{inputs.parameters.output-bucket}}'
accessKeySecret:
name: minio-credentials
key: accessKey
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{inputs.parameters.output-path}}/all.gz'
metadata: {}
container:
name: ''
image: scootna/hsextract:0.1
command:
- /bin/sh
- '-c'
args:
- >-
python3 hsextract/main.py extract /tmp/subset-data && mv
/tmp/subset-data/.hs /tmp/subset-data/hs
resources: {}
entrypoint: nwm1-subset-with-extraction-template
arguments:
parameters:
Expand Down
64 changes: 11 additions & 53 deletions app/argo/nwm2-subset-minio.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,18 @@ spec:
value: '{{workflow.parameters.output-path}}'
- name: output-bucket
value: '{{workflow.parameters.output-bucket}}'
- - name: metadata-extractor
template: metadata-extractor-template
- - name: metadata-extraction
templateRef:
name: metadata-extractor
template: metadata-extractor
arguments:
parameters:
- name: bucket
value: '{{workflow.parameters.output-bucket}}'
- name: input-path
value: '{{workflow.parameters.output-path}}'
- name: output-path
value: '{{workflow.parameters.output-path}}'
- name: output-bucket
value: '{{workflow.parameters.output-bucket}}'
- name: nwm2-subset-template
inputs:
parameters:
Expand All @@ -58,6 +62,8 @@ spec:
artifacts:
- name: subset-results
path: /srv/output
archive:
none: {}
s3:
endpoint: api.minio.cuahsi.io
bucket: '{{inputs.parameters.output-bucket}}'
Expand All @@ -67,7 +73,7 @@ spec:
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{inputs.parameters.output-path}}/subset.gz'
key: '{{inputs.parameters.output-path}}'
metadata: {}
container:
name: ''
Expand All @@ -88,54 +94,6 @@ spec:
cpu: '1'
ephemeral-storage: 2Gi
memory: 1Gi
- name: metadata-extractor-template
inputs:
parameters:
- name: output-path
- name: output-bucket
artifacts:
- name: extractor-input
path: /tmp/subset-data
s3:
endpoint: api.minio.cuahsi.io
bucket: '{{inputs.parameters.output-bucket}}'
accessKeySecret:
name: minio-credentials
key: accessKey
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{inputs.parameters.output-path}}/subset.gz'
outputs:
artifacts:
- name: extractor-output
path: /tmp/subset-data
s3:
endpoint: api.minio.cuahsi.io
bucket: '{{inputs.parameters.output-bucket}}'
accessKeySecret:
name: minio-credentials
key: accessKey
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{inputs.parameters.output-path}}/all.gz'
metadata: {}
container:
name: ''
image: scootna/hsextract:0.1
command:
- /bin/sh
- '-c'
args:
- >-
python3 hsextract/main.py extract /tmp/subset-data && mv
/tmp/subset-data/.hs /tmp/subset-data/hs
resources:
requests:
cpu: '1'
ephemeral-storage: 2Gi
memory: 1Gi
entrypoint: nwm2-subset-with-extraction-template
arguments:
parameters:
Expand Down
Loading

0 comments on commit 1e1d493

Please sign in to comment.