Skip to content

Commit

Permalink
update nwm2 and parflow workflows for configurable buckets
Browse files Browse the repository at this point in the history
  • Loading branch information
@sblack-usu
sblack-usu committed Jan 5, 2024
1 parent 8b97fbf commit 487fe82
Show file tree
Hide file tree
Showing 2 changed files with 53 additions and 76 deletions.
58 changes: 22 additions & 36 deletions app/argo/nwm2-subset-minio.yml
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,6 @@
metadata:
name: nwm2-subset-minio
generateName: nwm2-subset-minio-
namespace: workflows
uid: 482ae314-0f49-4cb7-a991-6c9c3546d092
resourceVersion: '293406895'
generation: 4
creationTimestamp: '2023-10-17T16:04:16Z'
labels:
workflows.argoproj.io/creator: '116317613255759624565'
managedFields:
- manager: argo
operation: Update
apiVersion: argoproj.io/v1alpha1
time: '2023-10-19T22:37:11Z'
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:generateName: {}
f:labels:
.: {}
f:workflows.argoproj.io/creator: {}
f:spec: {}
spec:
templates:
- name: nwm2-subset-with-extraction-template
Expand All @@ -40,22 +20,27 @@ spec:
value: '{{workflow.parameters.y_north}}'
- name: x_east
value: '{{workflow.parameters.x_east}}'
- name: artifact-path
value: '{{workflow.parameters.output_path}}'
- name: output-path
value: '{{workflow.parameters.output-path}}'
- name: output-bucket
value: '{{workflow.parameters.output-bucket}}'
- - name: metadata-extractor
template: metadata-extractor-template
arguments:
parameters:
- name: artifact-path
value: '{{workflow.parameters.output_path}}'
- name: output-path
value: '{{workflow.parameters.output-path}}'
- name: output-bucket
value: '{{workflow.parameters.output-bucket}}'
- name: nwm2-subset-template
inputs:
parameters:
- name: y_south
- name: x_west
- name: y_north
- name: x_east
- name: artifact-path
- name: output-path
- name: output-bucket
artifacts:
- name: nwm2-input-data
path: /srv/domain
Expand All @@ -75,14 +60,14 @@ spec:
path: /srv/output
s3:
endpoint: api.minio.cuahsi.io
bucket: subsetter-outputs
bucket: '{{inputs.parameters.output-bucket}}'
accessKeySecret:
name: minio-credentials
key: accessKey
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{inputs.parameters.artifact-path}}/subset.gz'
key: '{{inputs.parameters.output-path}}/subset.gz'
metadata: {}
container:
name: ''
Expand All @@ -106,34 +91,35 @@ spec:
- name: metadata-extractor-template
inputs:
parameters:
- name: artifact-path
- name: output-path
- name: output-bucket
artifacts:
- name: extractor-input
path: /tmp/subset-data
s3:
endpoint: api.minio.cuahsi.io
bucket: subsetter-outputs
bucket: '{{inputs.parameters.output-bucket}}'
accessKeySecret:
name: minio-credentials
key: accessKey
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{inputs.parameters.artifact-path}}/subset.gz'
key: '{{inputs.parameters.output-path}}/subset.gz'
outputs:
artifacts:
- name: extractor-output
path: /tmp/subset-data
s3:
endpoint: api.minio.cuahsi.io
bucket: subsetter-outputs
bucket: '{{inputs.parameters.output-bucket}}'
accessKeySecret:
name: minio-credentials
key: accessKey
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{inputs.parameters.artifact-path}}/all.gz'
key: '{{inputs.parameters.output-path}}/all.gz'
metadata: {}
container:
name: ''
Expand All @@ -153,10 +139,6 @@ spec:
entrypoint: nwm2-subset-with-extraction-template
arguments:
parameters:
- name: username
value: abc123
- name: path_guid
value: abc123
- name: y_south
value: '382582.18746'
- name: x_west
Expand All @@ -165,6 +147,10 @@ spec:
value: '367584.8784'
- name: x_east
value: '1734488.4526'
- name: output-path
value: 'asdf'
- name: output-bucket
value: 'subsetter-outputs'
templateDefaults:
inputs: {}
outputs: {}
Expand Down
71 changes: 31 additions & 40 deletions app/argo/parflow-subset-minio.yml
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,6 @@
metadata:
name: parflow-subset-v1-by-huc-minio
generateName: parflow-subset-v1-by-huc-minio-
namespace: workflows
uid: e11a1832-adff-42b1-99d7-7d20adb0ce0a
resourceVersion: '293388502'
generation: 15
creationTimestamp: '2023-10-10T15:00:11Z'
labels:
workflows.argoproj.io/creator: '116317613255759624565'
managedFields:
- manager: argo
operation: Update
apiVersion: argoproj.io/v1alpha1
time: '2023-10-19T22:11:02Z'
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:generateName: {}
f:labels:
.: {}
f:workflows.argoproj.io/creator: {}
f:spec: {}
spec:
templates:
- name: parflow-subset-huc
Expand All @@ -32,8 +12,10 @@ spec:
template: huc-to-shp
arguments:
parameters:
- name: artifact-path
value: '{{workflow.parameters.output_path}}'
- name: output-path
value: '{{workflow.parameters.output-path}}'
- name: output-bucket
value: '{{workflow.parameters.output-bucket}}'
- name: hucs
value: '{{workflow.parameters.hucs}}'
- - name: subset-parflow
Expand All @@ -42,33 +24,38 @@ spec:
parameters:
- name: label
value: '{{workflow.parameters.label}}'
- name: artifact-path
value: '{{workflow.parameters.output_path}}'
- name: output-path
value: '{{workflow.parameters.output-path}}'
- name: output-bucket
value: '{{workflow.parameters.output-bucket}}'
- - name: metadata-extractor
template: metadata-extractor
arguments:
parameters:
- name: artifact-path
value: '{{workflow.parameters.output_path}}'
- name: output-path
value: '{{workflow.parameters.output-path}}'
- name: output-bucket
value: '{{workflow.parameters.output-bucket}}'
- name: huc-to-shp
inputs:
parameters:
- name: artifact-path
- name: output-path
- name: output-bucket
- name: hucs
outputs:
artifacts:
- name: shape-results
path: /output
s3:
endpoint: api.minio.cuahsi.io
bucket: subsetter-outputs
bucket: '{{inputs.parameters.output-bucket}}'
accessKeySecret:
name: minio-credentials
key: accessKey
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{inputs.parameters.artifact-path}}/shape.gz'
key: '{{inputs.parameters.output-path}}/shape.gz'
metadata: {}
container:
name: ''
Expand All @@ -81,7 +68,8 @@ spec:
inputs:
parameters:
- name: label
- name: artifact-path
- name: output-path
- name: output-bucket
artifacts:
- name: pfinput-data
path: /srv/input
Expand All @@ -99,28 +87,28 @@ spec:
path: /srv/shape
s3:
endpoint: api.minio.cuahsi.io
bucket: subsetter-outputs
bucket: '{{inputs.parameters.output-bucket}}'
accessKeySecret:
name: minio-credentials
key: accessKey
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{inputs.parameters.artifact-path}}/shape.gz'
key: '{{inputs.parameters.output-path}}/shape.gz'
outputs:
artifacts:
- name: subset-results
path: /srv/shape
s3:
endpoint: api.minio.cuahsi.io
bucket: subsetter-outputs
bucket: '{{inputs.parameters.output-bucket}}'
accessKeySecret:
name: minio-credentials
key: accessKey
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{inputs.parameters.artifact-path}}/subset.gz'
key: '{{inputs.parameters.output-path}}/subset.gz'
metadata: {}
container:
name: ''
Expand All @@ -146,34 +134,35 @@ spec:
- name: metadata-extractor
inputs:
parameters:
- name: artifact-path
- name: output-path
- name: output-bucket
artifacts:
- name: extractor-input
path: /tmp/subset-data
s3:
endpoint: api.minio.cuahsi.io
bucket: subsetter-outputs
bucket: '{{inputs.parameters.output-bucket}}'
accessKeySecret:
name: minio-credentials
key: accessKey
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{inputs.parameters.artifact-path}}/subset.gz'
key: '{{inputs.parameters.output-path}}/subset.gz'
outputs:
artifacts:
- name: extractor-output
path: /tmp/subset-data
s3:
endpoint: api.minio.cuahsi.io
bucket: subsetter-outputs
bucket: '{{inputs.parameters.output-bucket}}'
accessKeySecret:
name: minio-credentials
key: accessKey
secretKeySecret:
name: minio-credentials
key: secretKey
key: '{{inputs.parameters.artifact-path}}/all.gz'
key: '{{inputs.parameters.output-path}}/all.gz'
metadata: {}
container:
name: ''
Expand All @@ -191,8 +180,10 @@ spec:
parameters:
- name: label
value: pf-subset-job
- name: output_path
- name: output-path
value: abc123
- name: output-bucket
value: subsetter-outputs
- name: hucs
value: 102600070103,102600070104,102600090205,102600050206,102600070102
templateDefaults:
Expand Down

0 comments on commit 487fe82

Please sign in to comment.