compress subsetter outputs

CUAHSI · Jan 7, 2024 · b8db299 · b8db299
1 parent 3c73406
commit b8db299
Show file tree

Hide file tree

Showing 5 changed files with 7 additions and 43 deletions.
diff --git a/app/api/subsetter/app/routers/storage/router.py b/app/api/subsetter/app/routers/storage/router.py
@@ -10,7 +10,7 @@
 async def presigned_get_minio(workflow_params: WorkflowDep):
     submission = workflow_params.user.get_submission(workflow_params.workflow_id)
     url = get_minio_client().presigned_get_object(
-        workflow_params.user.username, f"argo_workflows/{submission.workflow_name}/{submission.workflow_id}/all.gz"
+        workflow_params.user.username, f"argo_workflows/{submission.workflow_name}/{submission.workflow_id}"
     )
     return {'url': url}
 

diff --git a/app/argo/metadata-extraction.yml b/app/argo/metadata-extraction.yml
@@ -14,7 +14,7 @@ spec:
             value: '{{workflow.parameters.output-path}}'
         artifacts:
           - name: input-data
-            path: /tmp/input-data
+            path: /tmp/resource
             s3:
               endpoint: api.minio.cuahsi.io
               bucket: '{{inputs.parameters.bucket}}'
@@ -28,9 +28,7 @@ spec:
       outputs:
         artifacts:
           - name: output-metadata
-            path: /tmp/input-data
-            archive:
-              none: {}
+            path: /tmp/resource
             s3:
               endpoint: api.minio.cuahsi.io
               bucket: '{{inputs.parameters.bucket}}'
@@ -50,7 +48,7 @@ spec:
           - '-c'
         args:
           - >-
-            python3 hsextract/main.py extract /tmp/input-data
+            python3 hsextract/main.py extract /tmp/resource
         resources: {}
   entrypoint: metadata-extractor
   arguments:

diff --git a/app/argo/nwm1-subset-minio.yml b/app/argo/nwm1-subset-minio.yml
@@ -62,8 +62,6 @@ spec:
         artifacts:
           - name: subset-results
             path: /srv/output
-            archive:
-              none: {}
             s3:
               endpoint: api.minio.cuahsi.io
               bucket: '{{inputs.parameters.output-bucket}}'

diff --git a/app/argo/nwm2-subset-minio.yml b/app/argo/nwm2-subset-minio.yml
@@ -62,8 +62,6 @@ spec:
         artifacts:
           - name: subset-results
             path: /srv/output
-            archive:
-              none: {}
             s3:
               endpoint: api.minio.cuahsi.io
               bucket: '{{inputs.parameters.output-bucket}}'

diff --git a/app/argo/parflow-subset-minio.yml b/app/argo/parflow-subset-minio.yml
@@ -9,7 +9,9 @@ spec:
       metadata: {}
       steps:
         - - name: huc-to-shp
-            template: huc-to-shp
+            templateRef:
+              name: huc-to-shp
+              template: huc-to-shp
             arguments:
               parameters:
                 - name: output-path
@@ -40,36 +42,6 @@ spec:
                   value: '{{workflow.parameters.output-path}}'
                 - name: output-path
                   value: '{{workflow.parameters.output-path}}'
-    - name: huc-to-shp
-      inputs:
-        parameters:
-          - name: output-path
-          - name: output-bucket
-          - name: hucs
-      outputs:
-        artifacts:
-          - name: shape-results
-            path: /output
-            archive:
-              none: {}
-            s3:
-              endpoint: api.minio.cuahsi.io
-              bucket: '{{inputs.parameters.output-bucket}}'
-              accessKeySecret:
-                name: minio-credentials
-                key: accessKey
-              secretKeySecret:
-                name: minio-credentials
-                key: secretKey
-              key: '{{inputs.parameters.output-path}}'
-      metadata: {}
-      container:
-        name: ''
-        image: us-central1-docker.pkg.dev/apps-320517/subsetter/huc-to-shp:0.0.4
-        args:
-          - '{{inputs.parameters.hucs}}'
-          - /output/watershed.shp
-        resources: {}
     - name: parflow-subset-v1
       inputs:
         parameters:
@@ -105,8 +77,6 @@ spec:
         artifacts:
           - name: subset-results
             path: /srv/shape
-            archive:
-              none: {}
             s3:
               endpoint: api.minio.cuahsi.io
               bucket: '{{inputs.parameters.output-bucket}}'