From b923736746c43654455d815bb087e6b023c61572 Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Wed, 19 Oct 2022 14:12:29 -0500 Subject: [PATCH 1/8] Rewrite ingest and discovery process for large datasets. Add support for daily record datetime inference. Add CMIP6 ingests. --- .../nex-gddp-cmip6-crossover.json | 41 ++++ .../nex-gddp-cmip6-daily-hurs.json | 42 ++++ .../nex-gddp-cmip6-daily-huss.json | 42 ++++ .../nex-gddp-cmip6-daily-pr.json | 42 ++++ .../nex-gddp-cmip6-daily-rlds.json | 42 ++++ .../nex-gddp-cmip6-daily-rsds.json | 42 ++++ .../nex-gddp-cmip6-daily-sfcwind.json | 42 ++++ .../nex-gddp-cmip6-daily-tas.json | 42 ++++ .../nex-gddp-cmip6-daily-tasmax.json | 42 ++++ .../nex-gddp-cmip6-daily-tasmin.json | 42 ++++ .../p10/nex-gddp-cmip6-monthly-p10-hurs.json | 42 ++++ .../p10/nex-gddp-cmip6-monthly-p10-huss.json | 42 ++++ .../p10/nex-gddp-cmip6-monthly-p10-pr.json | 42 ++++ .../p10/nex-gddp-cmip6-monthly-p10-rlds.json | 42 ++++ .../p10/nex-gddp-cmip6-monthly-p10-rsds.json | 42 ++++ .../nex-gddp-cmip6-monthly-p10-sfcwind.json | 42 ++++ .../p10/nex-gddp-cmip6-monthly-p10-tas.json | 42 ++++ .../nex-gddp-cmip6-monthly-p10-tasmax.json | 42 ++++ .../nex-gddp-cmip6-monthly-p10-tasmin.json | 42 ++++ .../p50/nex-gddp-cmip6-monthly-p50-hurs.json | 42 ++++ .../p50/nex-gddp-cmip6-monthly-p50-huss.json | 42 ++++ .../p50/nex-gddp-cmip6-monthly-p50-pr.json | 42 ++++ .../p50/nex-gddp-cmip6-monthly-p50-rlds.json | 42 ++++ .../p50/nex-gddp-cmip6-monthly-p50-rsds.json | 42 ++++ .../nex-gddp-cmip6-monthly-p50-sfcwind.json | 42 ++++ .../p50/nex-gddp-cmip6-monthly-p50-tas.json | 42 ++++ .../nex-gddp-cmip6-monthly-p50-tasmax.json | 42 ++++ .../nex-gddp-cmip6-monthly-p50-tasmin.json | 42 ++++ .../p90/nex-gddp-cmip6-monthly-p90-hurs.json | 42 ++++ .../p90/nex-gddp-cmip6-monthly-p90-huss.json | 42 ++++ .../p90/nex-gddp-cmip6-monthly-p90-pr.json | 42 ++++ .../p90/nex-gddp-cmip6-monthly-p90-rlds.json | 42 ++++ .../p90/nex-gddp-cmip6-monthly-p90-rsds.json | 42 ++++ .../nex-gddp-cmip6-monthly-p90-sfcwind.json | 42 ++++ .../p90/nex-gddp-cmip6-monthly-p90-tas.json | 42 ++++ .../nex-gddp-cmip6-monthly-p90-tasmax.json | 42 ++++ .../nex-gddp-cmip6-monthly-p90-tasmin.json | 42 ++++ .../nex-gddp-cmip6-crossover.json | 22 ++ .../nex-gddp-cmip6-daily-historical.json | 218 ++++++++++++++++++ .../nex-gddp-cmip6-daily-ssp245-gfdl.json | 110 +++++++++ .../nex-gddp-cmip6-daily-ssp245-giss.json | 110 +++++++++ .../nex-gddp-cmip6-daily-ssp585-gfdl.json | 110 +++++++++ .../nex-gddp-cmip6-daily-ssp585-giss.json | 110 +++++++++ .../nex-gddp-cmip6-monthly-p10.json | 74 ++++++ .../nex-gddp-cmip6-monthly-p50.json | 74 ++++++ .../nex-gddp-cmip6-monthly-p90.json | 74 ++++++ data/step_function_inputs/test-ingest.json | 0 deploy/app.py | 5 + deploy/cdk/lambda_stack.py | 6 + deploy/cdk/step_function_stack.py | 41 +++- lambdas/build-stac/utils/events.py | 2 +- lambdas/build-stac/utils/regex.py | 8 + lambdas/cogify/handler.py | 7 +- lambdas/discovery-trigger/handler.py | 17 ++ lambdas/discovery-trigger/requirements.txt | 2 + lambdas/s3-discovery/handler.py | 74 +++--- scripts/item.py | 12 +- scripts/utils.py | 5 +- 58 files changed, 2582 insertions(+), 52 deletions(-) create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-crossover.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-hurs.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-huss.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-pr.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-rlds.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-rsds.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-sfcwind.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-tas.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-tasmax.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-tasmin.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-hurs.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-huss.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-pr.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-rlds.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-rsds.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-sfcwind.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-tas.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-tasmax.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-tasmin.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-hurs.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-huss.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-pr.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-rlds.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-rsds.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-sfcwind.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-tas.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-tasmax.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-tasmin.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-hurs.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-huss.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-pr.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-rlds.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-rsds.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-sfcwind.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-tas.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-tasmax.json create mode 100644 data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-tasmin.json create mode 100644 data/step_function_inputs/nex-gddp-cmip6-crossover.json create mode 100644 data/step_function_inputs/nex-gddp-cmip6-daily-historical.json create mode 100644 data/step_function_inputs/nex-gddp-cmip6-daily-ssp245-gfdl.json create mode 100644 data/step_function_inputs/nex-gddp-cmip6-daily-ssp245-giss.json create mode 100644 data/step_function_inputs/nex-gddp-cmip6-daily-ssp585-gfdl.json create mode 100644 data/step_function_inputs/nex-gddp-cmip6-daily-ssp585-giss.json create mode 100644 data/step_function_inputs/nex-gddp-cmip6-monthly-p10.json create mode 100644 data/step_function_inputs/nex-gddp-cmip6-monthly-p50.json create mode 100644 data/step_function_inputs/nex-gddp-cmip6-monthly-p90.json create mode 100644 data/step_function_inputs/test-ingest.json create mode 100644 lambdas/discovery-trigger/handler.py create mode 100644 lambdas/discovery-trigger/requirements.txt diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-crossover.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-crossover.json new file mode 100644 index 00000000..49ad4846 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-crossover.json @@ -0,0 +1,41 @@ +{ + "id": "nex-gddp-cmip6-crossover", + "type": "Collection", + "links": [], + "title": "CMIP6 2-degree average temperature crossover predictions", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "2015-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "Predicted year at which the average daily temperature has risen by 2-degrees", + "stac_version": "1.0.0", + "dashboard:is_periodic": false, + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-hurs.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-hurs.json new file mode 100644 index 00000000..67dee215 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-hurs.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-hurs", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected near-surface relative humidity", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected near-surface relative humidity", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-huss.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-huss.json new file mode 100644 index 00000000..92ae2af8 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-huss.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-huss", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected near-surface specific humidity", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected near-surface specific humidity", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-pr.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-pr.json new file mode 100644 index 00000000..6e2fa3cb --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-pr.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-pr", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected precipitation", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected precipitation", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-rlds.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-rlds.json new file mode 100644 index 00000000..e6521342 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-rlds.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-rlds", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected surface downwelling longwave radiation", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected surface downwelling longwave radiation", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-rsds.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-rsds.json new file mode 100644 index 00000000..27318ee7 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-rsds.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-rdsds", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected surface downwelling shortwave radiation", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected surface downwelling shortwave radiation", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-sfcwind.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-sfcwind.json new file mode 100644 index 00000000..e079ebed --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-sfcwind.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-sfcwind", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected near-surface wind speed", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected near-surface wind speed", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-tas.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-tas.json new file mode 100644 index 00000000..6a2c1fd2 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-tas.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-tas", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected daily near-surface air temperature", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected daily near-surface air temperature", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-tasmax.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-tasmax.json new file mode 100644 index 00000000..74cb8197 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-tasmax.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-tasmax", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected daily maximum near-surface air temperature", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected daily maximum near-surface air temperature", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-tasmin.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-tasmin.json new file mode 100644 index 00000000..fc634c20 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-daily/nex-gddp-cmip6-daily-tasmin.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-tasmin", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected daily minimum near-surface air temperature", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected daily minimum near-surface air temperature", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-hurs.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-hurs.json new file mode 100644 index 00000000..f5bda677 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-hurs.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-hurs", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected near-surface relative humidity", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected near-surface relative humidity", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "month", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-huss.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-huss.json new file mode 100644 index 00000000..92ae2af8 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-huss.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-huss", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected near-surface specific humidity", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected near-surface specific humidity", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-pr.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-pr.json new file mode 100644 index 00000000..6e2fa3cb --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-pr.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-pr", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected precipitation", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected precipitation", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-rlds.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-rlds.json new file mode 100644 index 00000000..e6521342 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-rlds.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-rlds", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected surface downwelling longwave radiation", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected surface downwelling longwave radiation", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-rsds.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-rsds.json new file mode 100644 index 00000000..27318ee7 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-rsds.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-rdsds", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected surface downwelling shortwave radiation", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected surface downwelling shortwave radiation", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-sfcwind.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-sfcwind.json new file mode 100644 index 00000000..e079ebed --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-sfcwind.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-sfcwind", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected near-surface wind speed", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected near-surface wind speed", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-tas.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-tas.json new file mode 100644 index 00000000..6a2c1fd2 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-tas.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-tas", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected daily near-surface air temperature", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected daily near-surface air temperature", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-tasmax.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-tasmax.json new file mode 100644 index 00000000..74cb8197 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-tasmax.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-tasmax", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected daily maximum near-surface air temperature", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected daily maximum near-surface air temperature", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-tasmin.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-tasmin.json new file mode 100644 index 00000000..fc634c20 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-tasmin.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-tasmin", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected daily minimum near-surface air temperature", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected daily minimum near-surface air temperature", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-hurs.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-hurs.json new file mode 100644 index 00000000..67dee215 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-hurs.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-hurs", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected near-surface relative humidity", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected near-surface relative humidity", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-huss.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-huss.json new file mode 100644 index 00000000..92ae2af8 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-huss.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-huss", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected near-surface specific humidity", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected near-surface specific humidity", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-pr.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-pr.json new file mode 100644 index 00000000..6e2fa3cb --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-pr.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-pr", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected precipitation", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected precipitation", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-rlds.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-rlds.json new file mode 100644 index 00000000..e6521342 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-rlds.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-rlds", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected surface downwelling longwave radiation", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected surface downwelling longwave radiation", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-rsds.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-rsds.json new file mode 100644 index 00000000..27318ee7 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-rsds.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-rdsds", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected surface downwelling shortwave radiation", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected surface downwelling shortwave radiation", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-sfcwind.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-sfcwind.json new file mode 100644 index 00000000..e079ebed --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-sfcwind.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-sfcwind", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected near-surface wind speed", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected near-surface wind speed", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-tas.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-tas.json new file mode 100644 index 00000000..6a2c1fd2 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-tas.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-tas", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected daily near-surface air temperature", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected daily near-surface air temperature", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-tasmax.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-tasmax.json new file mode 100644 index 00000000..74cb8197 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-tasmax.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-tasmax", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected daily maximum near-surface air temperature", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected daily maximum near-surface air temperature", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-tasmin.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-tasmin.json new file mode 100644 index 00000000..7d42adbf --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-tasmin.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-monthly-p50-tasmin", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected monthly minimum near-surface air temperature", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected monthly minimum near-surface air temperature", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-hurs.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-hurs.json new file mode 100644 index 00000000..6858cb92 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-hurs.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-monthly-p90-hurs", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected near-surface relative humidity", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected near-surface relative humidity", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-huss.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-huss.json new file mode 100644 index 00000000..92ae2af8 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-huss.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-huss", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected near-surface specific humidity", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected near-surface specific humidity", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-pr.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-pr.json new file mode 100644 index 00000000..6e2fa3cb --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-pr.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-pr", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected precipitation", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected precipitation", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-rlds.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-rlds.json new file mode 100644 index 00000000..e6521342 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-rlds.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-rlds", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected surface downwelling longwave radiation", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected surface downwelling longwave radiation", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-rsds.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-rsds.json new file mode 100644 index 00000000..27318ee7 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-rsds.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-rdsds", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected surface downwelling shortwave radiation", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected surface downwelling shortwave radiation", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-sfcwind.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-sfcwind.json new file mode 100644 index 00000000..e079ebed --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-sfcwind.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-sfcwind", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected near-surface wind speed", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected near-surface wind speed", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-tas.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-tas.json new file mode 100644 index 00000000..6a2c1fd2 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-tas.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-tas", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected daily near-surface air temperature", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected daily near-surface air temperature", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-tasmax.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-tasmax.json new file mode 100644 index 00000000..74cb8197 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-tasmax.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-tasmax", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected daily maximum near-surface air temperature", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected daily maximum near-surface air temperature", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-tasmin.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-tasmin.json new file mode 100644 index 00000000..fc634c20 --- /dev/null +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-tasmin.json @@ -0,0 +1,42 @@ +{ + "id": "nex-gddp-cmip6-daily-tasmin", + "type": "Collection", + "links": [], + "title": "CMIP6 historical and projected daily minimum near-surface air temperature", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "1950-01-01T00:00:00Z", + "2101-01-01T00:00:00Z" + ] + ] + } + }, + "license": "MIT", + "description": "CMIP6 historical and projected daily minimum near-surface air temperature", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + } +} diff --git a/data/step_function_inputs/nex-gddp-cmip6-crossover.json b/data/step_function_inputs/nex-gddp-cmip6-crossover.json new file mode 100644 index 00000000..bfda8fd5 --- /dev/null +++ b/data/step_function_inputs/nex-gddp-cmip6-crossover.json @@ -0,0 +1,22 @@ +[ + { + "collection": "nex-gddp-cmip6-crossover", + "prefix": "crossing/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "tas_CrossingYear_CMIP6_ssp245.tif", + "discovery": "s3", + "properties": { + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-crossover", + "prefix": "crossing/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "tas_CrossingYear_CMIP6_ssp585.tif", + "discovery": "s3", + "properties": { + "ssp": "585" + } + } +] \ No newline at end of file diff --git a/data/step_function_inputs/nex-gddp-cmip6-daily-historical.json b/data/step_function_inputs/nex-gddp-cmip6-daily-historical.json new file mode 100644 index 00000000..4bb2e910 --- /dev/null +++ b/data/step_function_inputs/nex-gddp-cmip6-daily-historical.json @@ -0,0 +1,218 @@ +[ + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/hurs/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GFDL-CM4", + "variable": "hurs" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/huss/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GFDL-CM4", + "variable": "huss" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/pr/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GFDL-CM4", + "variable": "pr" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/rlds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GFDL-CM4", + "variable": "rlds" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/rsds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GFDL-CM4", + "variable": "rsds" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/sfcWind/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GFDL-CM4", + "variable": "sfcWind" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/tasmax/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GFDL-CM4", + "variable": "tasmax" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/tasmin/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GFDL-CM4", + "variable": "tasmin" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/tas/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GFDL-CM4", + "variable": "tas" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/hurs/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GISS-E2.1-G", + "variable": "hurs" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/huss/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GISS-E2.1-G", + "variable": "huss" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/pr/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GISS-E2.1-G", + "variable": "pr" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/rlds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GISS-E2.1-G", + "variable": "rlds" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/rsds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GISS-E2.1-G", + "variable": "rsds" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/sfcWind/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GISS-E2.1-G", + "variable": "sfcWind" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/tasmax/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GISS-E2.1-G", + "variable": "tasmax" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/tasmin/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GISS-E2.1-G", + "variable": "tasmin" + } + }, + { + "collection": "nex-gddp-cmip6-daily", + "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/tas/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "properties": { + "ssp": "historical", + "model": "GISS-E2.1-G", + "variable": "tas" + } + } +] \ No newline at end of file diff --git a/data/step_function_inputs/nex-gddp-cmip6-daily-ssp245-gfdl.json b/data/step_function_inputs/nex-gddp-cmip6-daily-ssp245-gfdl.json new file mode 100644 index 00000000..c132afb5 --- /dev/null +++ b/data/step_function_inputs/nex-gddp-cmip6-daily-ssp245-gfdl.json @@ -0,0 +1,110 @@ +[ + { + "collection": "nex-gddp-cmip6-daily-hurs", + "prefix": "daily/GFDL-CM4/ssp245/r1i1p1f1/hurs/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-daily-huss", + "prefix": "daily/GFDL-CM4/ssp245/r1i1p1f1/huss/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-daily-pr", + "prefix": "daily/GFDL-CM4/ssp245/r1i1p1f1/pr/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-daily-rlds", + "prefix": "daily/GFDL-CM4/ssp245/r1i1p1f1/rlds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-daily-rsds", + "prefix": "daily/GFDL-CM4/ssp245/r1i1p1f1/rsds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-daily-sfcwind", + "prefix": "daily/GFDL-CM4/ssp245/r1i1p1f1/sfcWind/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-daily-tasmax", + "prefix": "daily/GFDL-CM4/ssp245/r1i1p1f1/tasmax/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-daily-tasmin", + "prefix": "daily/GFDL-CM4/ssp245/r1i1p1f1/tasmin/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-daily-tas", + "prefix": "daily/GFDL-CM4/ssp245/r1i1p1f1/tas/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "245" + } + } +] \ No newline at end of file diff --git a/data/step_function_inputs/nex-gddp-cmip6-daily-ssp245-giss.json b/data/step_function_inputs/nex-gddp-cmip6-daily-ssp245-giss.json new file mode 100644 index 00000000..f04b5371 --- /dev/null +++ b/data/step_function_inputs/nex-gddp-cmip6-daily-ssp245-giss.json @@ -0,0 +1,110 @@ +[ + { + "collection": "nex-gddp-cmip6-daily-hurs", + "prefix": "daily/GISS-E2-1-G/ssp245/r1i1p1f1/hurs/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-daily-huss", + "prefix": "daily/GISS-E2-1-G/ssp245/r1i1p1f1/huss/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-daily-pr", + "prefix": "daily/GISS-E2-1-G/ssp245/r1i1p1f1/pr/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-daily-rlds", + "prefix": "daily/GISS-E2-1-G/ssp245/r1i1p1f1/rlds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-daily-rsds", + "prefix": "daily/GISS-E2-1-G/ssp245/r1i1p1f1/rsds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-daily-sfcwind", + "prefix": "daily/GISS-E2-1-G/ssp245/r1i1p1f1/sfcWind/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-daily-tasmax", + "prefix": "daily/GISS-E2-1-G/ssp245/r1i1p1f1/tasmax/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-daily-tasmin", + "prefix": "daily/GISS-E2-1-G/ssp245/r1i1p1f1/tasmin/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "245" + } + }, + { + "collection": "nex-gddp-cmip6-daily-tas", + "prefix": "daily/GISS-E2-1-G/ssp245/r1i1p1f1/tas/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "245" + } + } +] \ No newline at end of file diff --git a/data/step_function_inputs/nex-gddp-cmip6-daily-ssp585-gfdl.json b/data/step_function_inputs/nex-gddp-cmip6-daily-ssp585-gfdl.json new file mode 100644 index 00000000..f41e78a6 --- /dev/null +++ b/data/step_function_inputs/nex-gddp-cmip6-daily-ssp585-gfdl.json @@ -0,0 +1,110 @@ +[ + { + "collection": "nex-gddp-cmip6-daily-hurs", + "prefix": "daily/GFDL-CM4/ssp585/r1i1p1f1/hurs/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "585" + } + }, + { + "collection": "nex-gddp-cmip6-daily-huss", + "prefix": "daily/GFDL-CM4/ssp585/r1i1p1f1/huss/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "585" + } + }, + { + "collection": "nex-gddp-cmip6-daily-pr", + "prefix": "daily/GFDL-CM4/ssp585/r1i1p1f1/pr/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "585" + } + }, + { + "collection": "nex-gddp-cmip6-daily-rlds", + "prefix": "daily/GFDL-CM4/ssp585/r1i1p1f1/rlds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "585" + } + }, + { + "collection": "nex-gddp-cmip6-daily-rsds", + "prefix": "daily/GFDL-CM4/ssp585/r1i1p1f1/rsds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "585" + } + }, + { + "collection": "nex-gddp-cmip6-daily-sfcwind", + "prefix": "daily/GFDL-CM4/ssp585/r1i1p1f1/sfcWind/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "585" + } + }, + { + "collection": "nex-gddp-cmip6-daily-tasmax", + "prefix": "daily/GFDL-CM4/ssp585/r1i1p1f1/tasmax/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "585" + } + }, + { + "collection": "nex-gddp-cmip6-daily-tasmin", + "prefix": "daily/GFDL-CM4/ssp585/r1i1p1f1/tasmin/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "585" + } + }, + { + "collection": "nex-gddp-cmip6-daily-tas", + "prefix": "daily/GFDL-CM4/ssp585/r1i1p1f1/tas/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GFDL-CM4", + "ssp": "585" + } + } +] \ No newline at end of file diff --git a/data/step_function_inputs/nex-gddp-cmip6-daily-ssp585-giss.json b/data/step_function_inputs/nex-gddp-cmip6-daily-ssp585-giss.json new file mode 100644 index 00000000..e1d65ac9 --- /dev/null +++ b/data/step_function_inputs/nex-gddp-cmip6-daily-ssp585-giss.json @@ -0,0 +1,110 @@ +[ + { + "collection": "nex-gddp-cmip6-daily-hurs", + "prefix": "daily/GISS-E2-1-G/ssp585/r1i1p1f1/hurs/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "585" + } + }, + { + "collection": "nex-gddp-cmip6-daily-huss", + "prefix": "daily/GISS-E2-1-G/ssp585/r1i1p1f1/huss/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "585" + } + }, + { + "collection": "nex-gddp-cmip6-daily-pr", + "prefix": "daily/GISS-E2-1-G/ssp585/r1i1p1f1/pr/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "585" + } + }, + { + "collection": "nex-gddp-cmip6-daily-rlds", + "prefix": "daily/GISS-E2-1-G/ssp585/r1i1p1f1/rlds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "585" + } + }, + { + "collection": "nex-gddp-cmip6-daily-rsds", + "prefix": "daily/GISS-E2-1-G/ssp585/r1i1p1f1/rsds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "585" + } + }, + { + "collection": "nex-gddp-cmip6-daily-sfcwind", + "prefix": "daily/GISS-E2-1-G/ssp585/r1i1p1f1/sfcWind/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "585" + } + }, + { + "collection": "nex-gddp-cmip6-daily-tasmax", + "prefix": "daily/GISS-E2-1-G/ssp585/r1i1p1f1/tasmax/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "585" + } + }, + { + "collection": "nex-gddp-cmip6-daily-tasmin", + "prefix": "daily/GISS-E2-1-G/ssp585/r1i1p1f1/tasmin/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "585" + } + }, + { + "collection": "nex-gddp-cmip6-daily-tas", + "prefix": "daily/GISS-E2-1-G/ssp585/r1i1p1f1/tas/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "day", + "properties": { + "model": "GISS-E2.1-G", + "ssp": "585" + } + } +] \ No newline at end of file diff --git a/data/step_function_inputs/nex-gddp-cmip6-monthly-p10.json b/data/step_function_inputs/nex-gddp-cmip6-monthly-p10.json new file mode 100644 index 00000000..8873031c --- /dev/null +++ b/data/step_function_inputs/nex-gddp-cmip6-monthly-p10.json @@ -0,0 +1,74 @@ +[ + { + "collection": "nex-gddp-cmip6-monthly-p10-hurs", + "prefix": "monthly/CMIP6_ensemble_p10/hurs/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p10-huss", + "prefix": "monthly/CMIP6_ensemble_p10/huss/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p10-pr", + "prefix": "monthly/CMIP6_ensemble_p10/pr/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p10-rlds", + "prefix": "monthly/CMIP6_ensemble_p10/rlds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p10-rds", + "prefix": "monthly/CMIP6_ensemble_p10/rsds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p10-sfcwind", + "prefix": "monthly/CMIP6_ensemble_p10/sfcWind/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p10-tasmax", + "prefix": "monthly/CMIP6_ensemble_p10/tasmax/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p10-tasmin", + "prefix": "monthly/CMIP6_ensemble_p10/tasmin/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p10-tas", + "prefix": "monthly/CMIP6_ensemble_p10/tas/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + } +] \ No newline at end of file diff --git a/data/step_function_inputs/nex-gddp-cmip6-monthly-p50.json b/data/step_function_inputs/nex-gddp-cmip6-monthly-p50.json new file mode 100644 index 00000000..ebaf9bb6 --- /dev/null +++ b/data/step_function_inputs/nex-gddp-cmip6-monthly-p50.json @@ -0,0 +1,74 @@ +[ + { + "collection": "nex-gddp-cmip6-monthly-p50-hurs", + "prefix": "monthly/CMIP6_ensemble_median/hurs/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p50-huss", + "prefix": "monthly/CMIP6_ensemble_median/huss/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p50-pr", + "prefix": "monthly/CMIP6_ensemble_median/pr/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p50-rlds", + "prefix": "monthly/CMIP6_ensemble_median/rlds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p50-rds", + "prefix": "monthly/CMIP6_ensemble_median/rsds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p50-sfcwind", + "prefix": "monthly/CMIP6_ensemble_median/sfcWind/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p50-tasmax", + "prefix": "monthly/CMIP6_ensemble_median/tasmax/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p50-tasmin", + "prefix": "monthly/CMIP6_ensemble_median/tasmin/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p50-tas", + "prefix": "monthly/CMIP6_ensemble_median/tas/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + } +] \ No newline at end of file diff --git a/data/step_function_inputs/nex-gddp-cmip6-monthly-p90.json b/data/step_function_inputs/nex-gddp-cmip6-monthly-p90.json new file mode 100644 index 00000000..1c8d8f22 --- /dev/null +++ b/data/step_function_inputs/nex-gddp-cmip6-monthly-p90.json @@ -0,0 +1,74 @@ +[ + { + "collection": "nex-gddp-cmip6-monthly-p90-hurs", + "prefix": "monthly/CMIP6_ensemble_p90/hurs/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p90-huss", + "prefix": "monthly/CMIP6_ensemble_p90/huss/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p90-pr", + "prefix": "monthly/CMIP6_ensemble_p90/pr/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p90-rlds", + "prefix": "monthly/CMIP6_ensemble_p90/rlds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p90-rds", + "prefix": "monthly/CMIP6_ensemble_p90/rsds/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p90-sfcwind", + "prefix": "monthly/CMIP6_ensemble_p90/sfcWind/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p90-tasmax", + "prefix": "monthly/CMIP6_ensemble_p90/tasmax/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p90-tasmin", + "prefix": "monthly/CMIP6_ensemble_p90/tasmin/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + }, + { + "collection": "nex-gddp-cmip6-monthly-p90-tas", + "prefix": "monthly/CMIP6_ensemble_p90/tas/", + "bucket": "nex-gddp-cmip6-cog", + "filename_regex": "^.*.tif", + "discovery": "s3", + "datetime_range": "month" + } +] \ No newline at end of file diff --git a/data/step_function_inputs/test-ingest.json b/data/step_function_inputs/test-ingest.json new file mode 100644 index 00000000..e69de29b diff --git a/deploy/app.py b/deploy/app.py index eeaba96b..61826393 100644 --- a/deploy/app.py +++ b/deploy/app.py @@ -40,6 +40,7 @@ # Need to build arn manually otherwise it'll result in cyclic dependency cogify_arn = step_function_stack.build_arn(env_details, "cogify") pub_arn = step_function_stack.build_arn(env_details, "publication") +discover_arn = step_function_stack.build_arn(env_details, "discover") lambda_stack.grant_execution_privileges( lambda_function=lambda_stack.trigger_cogify_lambda, @@ -49,5 +50,9 @@ lambda_function=lambda_stack.trigger_ingest_lambda, workflow_arn=pub_arn, ) +lambda_stack.grant_execution_privileges( + lambda_function=lambda_stack.trigger_discovery_lambda, + workflow_arn=discover_arn, +) app.synth() diff --git a/deploy/cdk/lambda_stack.py b/deploy/cdk/lambda_stack.py index 80519c55..9634a57c 100644 --- a/deploy/cdk/lambda_stack.py +++ b/deploy/cdk/lambda_stack.py @@ -57,6 +57,12 @@ def __init__(self, app, construct_id, **kwargs) -> None: }, ) + # Proxy lambda to trigger discovery step function + self.trigger_discovery_lambda = self._python_lambda( + f"{construct_id}-trigger-discover-fn", + "../lambdas/discovery-trigger", + ) + # Proxy lambda to trigger cogify step function self.trigger_cogify_lambda = self._python_lambda( f"{construct_id}-trigger-cogify-fn", diff --git a/deploy/cdk/step_function_stack.py b/deploy/cdk/step_function_stack.py index 4033ed21..fe7d2168 100644 --- a/deploy/cdk/step_function_stack.py +++ b/deploy/cdk/step_function_stack.py @@ -1,3 +1,4 @@ +from re import S from typing import TYPE_CHECKING from aws_cdk import ( core, @@ -57,6 +58,16 @@ def _discovery_workflow( lambda_stack: "LambdaStack", queue_stack: "QueueStack", ) -> stepfunctions.StateMachine: + trigger_s3_discovery_task = self._lambda_task( + "Trigger new discovery state machine", + lambda_stack.trigger_discovery_lambda, + ) + + trigger_s3_discovery_task.add_retry( + interval=core.Duration.seconds(2), + max_attempts=5, + ) + s3_discovery_task = self._lambda_task( "S3 Discover Task", lambda_stack.s3_discovery_lambda, @@ -77,6 +88,10 @@ def _discovery_workflow( queue=queue_stack.stac_ready_queue, ) + maybe_next_discovery = stepfunctions.Choice(self, "NextDiscovery?").otherwise( + stepfunctions.Succeed(self, "Successful Ingest") + ) + maybe_cogify = ( stepfunctions.Choice(self, "Cogify?") .when( @@ -84,17 +99,25 @@ def _discovery_workflow( stepfunctions.Map( self, "Run concurrent queueing to cogify queue", - max_concurrency=100, + max_concurrency=3, items_path=stepfunctions.JsonPath.string_at("$.Payload.objects"), - ).iterator(enqueue_cogify_task), + result_path=stepfunctions.JsonPath.DISCARD, + output_path="$.Payload", + ) + .iterator(enqueue_cogify_task) + .next(maybe_next_discovery), ) .otherwise( stepfunctions.Map( self, "Run concurrent queueing to stac ready queue", - max_concurrency=100, + max_concurrency=3, items_path=stepfunctions.JsonPath.string_at("$.Payload.objects"), - ).iterator(enqueue_ready_task) + result_path=stepfunctions.JsonPath.DISCARD, + output_path="$.Payload", + ) + .iterator(enqueue_ready_task) + .next(maybe_next_discovery) ) ) @@ -111,6 +134,12 @@ def _discovery_workflow( .otherwise(stepfunctions.Fail(self, "Discovery Type not supported")) ) + # Defined below workflow to avoid circular dependency of steps + maybe_next_discovery.when( + stepfunctions.Condition.is_present("$.start_after"), + trigger_s3_discovery_task, + ) + return stepfunctions.StateMachine( self, "discover-sf", @@ -137,7 +166,7 @@ def _cogify_workflow( cogify_workflow = stepfunctions.Map( self, "Run concurrent cogifications", - max_concurrency=100, + max_concurrency=3, items_path=stepfunctions.JsonPath.string_at("$"), ).iterator(cogify_task.next(enqueue_task)) @@ -180,7 +209,7 @@ def _publication_workflow( build_and_submit_stac_items = stepfunctions.Map( self, "Submit to STAC Ingestor", - max_concurrency=100, + max_concurrency=3, items_path=stepfunctions.JsonPath.string_at("$"), ).iterator(build_stac_item_task.next(submit_stac_item_task)) diff --git a/lambdas/build-stac/utils/events.py b/lambdas/build-stac/utils/events.py index 9c547a06..486fe382 100644 --- a/lambdas/build-stac/utils/events.py +++ b/lambdas/build-stac/utils/events.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Dict, List, Literal, Optional, Union +from typing import Dict, List, Literal, Optional, Tuple, Union from pydantic import BaseModel, Field import pystac diff --git a/lambdas/build-stac/utils/regex.py b/lambdas/build-stac/utils/regex.py index 2257f5fe..86d90d99 100644 --- a/lambdas/build-stac/utils/regex.py +++ b/lambdas/build-stac/utils/regex.py @@ -21,9 +21,16 @@ def _calculate_month_range(datetime_obj: datetime) -> DATERANGE: return start_datetime, end_datetime +def _calculate_day_range(datetime_obj: datetime) -> DATERANGE: + start_datetime = datetime_obj + end_datetime = datetime_obj + relativedelta(hour=23, minute=59, second=59) + return start_datetime, end_datetime + + DATETIME_RANGE_METHODS: Dict[events.INTERVAL, Callable[[datetime], DATERANGE]] = { "month": _calculate_month_range, "year": _calculate_year_range, + "day": _calculate_day_range, } @@ -35,6 +42,7 @@ def extract_dates( """ DATE_REGEX_STRATEGIES = [ (r"_(\d{4}-\d{2}-\d{2})", "%Y-%m-%d"), + (r"_(\d{4}_\d{2}_\d{2})", "%Y_%m_%d"), (r"_(\d{8})", "%Y%m%d"), (r"_(\d{6})", "%Y%m"), (r"_(\d{4})", "%Y"), diff --git a/lambdas/cogify/handler.py b/lambdas/cogify/handler.py index b7589366..97d19d50 100644 --- a/lambdas/cogify/handler.py +++ b/lambdas/cogify/handler.py @@ -175,7 +175,11 @@ def handler(event, context): to_cog_config["filename"] = downloaded_filename to_cog_config["collection"] = collection - return_obj = {"granule_id": event["granule_id"], "collection": event["collection"]} + return_obj = { + "granule_id": event["granule_id"], + "collection": event["collection"], + "properties": event["properties"], + } output_locations = to_cog(upload=event.get("upload", False), **to_cog_config) @@ -188,6 +192,7 @@ def handler(event, context): if __name__ == "__main__": sample_event = { "collection": "OMDOAO3e", + "properties": {}, "href": "https://acdisc.gesdisc.eosdis.nasa.gov/data//Aura_OMI_Level3/OMDOAO3e.003/2022/OMI-Aura_L3-OMDOAO3e_2022m0120_v003-2022m0122t021759.he5", "upload": False, "granule_id": "G2205784904-GES_DISC", diff --git a/lambdas/discovery-trigger/handler.py b/lambdas/discovery-trigger/handler.py new file mode 100644 index 00000000..a0612a07 --- /dev/null +++ b/lambdas/discovery-trigger/handler.py @@ -0,0 +1,17 @@ +import os +import json + +import boto3 + + +def handler(event, context): + STEP_FUNCTION_ARN = os.environ["STEP_FUNCTION_ARN"] + + event.pop("objects", None) + + client = boto3.client("stepfunctions") + client.start_execution( + stateMachineArn=STEP_FUNCTION_ARN, + input=json.dumps(event), + ) + return diff --git a/lambdas/discovery-trigger/requirements.txt b/lambdas/discovery-trigger/requirements.txt new file mode 100644 index 00000000..53c8a6a5 --- /dev/null +++ b/lambdas/discovery-trigger/requirements.txt @@ -0,0 +1,2 @@ +boto3 +aws-lambda-powertools diff --git a/lambdas/s3-discovery/handler.py b/lambdas/s3-discovery/handler.py index fa654d70..1b25a1b1 100644 --- a/lambdas/s3-discovery/handler.py +++ b/lambdas/s3-discovery/handler.py @@ -1,3 +1,4 @@ +import json import os import re @@ -13,8 +14,14 @@ def assume_role(role_arn, session_name): return creds["Credentials"] -def list_bucket(bucket, prefix, filename_regex): - kwargs = {} +def handler(event, context): + bucket = event.get("bucket") + prefix = event.get("prefix", "") + filename_regex = event.get("filename_regex", None) + collection = event.get("collection", prefix.rstrip("/")) + properties = event.get("properties", {}) + cogify = event.pop("cogify", False) + if role_arn := os.environ.get("EXTERNAL_ROLE_ARN"): creds = assume_role(role_arn, "veda-data-pipelines_s3-discovery") kwargs = { @@ -22,47 +29,38 @@ def list_bucket(bucket, prefix, filename_regex): "aws_secret_access_key": creds["SecretAccessKey"], "aws_session_token": creds["SessionToken"], } - s3 = boto3.resource("s3", **kwargs) - try: - files = [] - bucket = s3.Bucket(bucket) - for obj in bucket.objects.filter(Prefix=prefix): - if filename_regex: - if re.match(filename_regex, obj.key): - files.append(obj.key) - else: - files.append(obj.key) - return files - - except: - print("Failed during s3 item/asset discovery") - raise - - -def handler(event, context): - bucket = event.pop("bucket") - prefix = event.pop("prefix", "") - - filenames = list_bucket( - bucket=bucket, prefix=prefix, filename_regex=event.pop("filename_regex", None) - ) + s3client = boto3.client("s3", **kwargs) + s3paginator = s3client.get_paginator("list_objects_v2") + start_after = event.pop("start_after", None) + if start_after: + pages = s3paginator.paginate( + Bucket=bucket, Prefix=prefix, StartAfter=start_after + ) + else: + pages = s3paginator.paginate(Bucket=bucket, Prefix=prefix) - files_objs = [] - cogify = event.pop("cogify", False) - collection = event.get("collection", prefix.rstrip("/")) - for filename in filenames: - files_objs.append( - { - **event, + file_objs_size = 0 + payload = {**event, "cogify": cogify, "objects": []} + for page in pages: + for obj in page["Contents"]: + # The limit is advertised at 256000, but we'll preserve some breathing room + if file_objs_size > 230000: + payload["start_after"] = start_after + break + filename = obj["Key"] + if filename_regex and not re.match(filename_regex, filename): + continue + file_obj = { "collection": collection, "s3_filename": f"s3://{bucket}/{filename}", "upload": event.get("upload", False), + "properties": properties, } - ) - return { - "cogify": cogify, - "objects": files_objs, - } + payload["objects"].append(file_obj) + file_obj_size = len(json.dumps(file_obj, ensure_ascii=False).encode("utf8")) + file_objs_size = file_objs_size + file_obj_size + start_after = filename + return payload if __name__ == "__main__": diff --git a/scripts/item.py b/scripts/item.py index 0d01ca01..9b06e6c9 100644 --- a/scripts/item.py +++ b/scripts/item.py @@ -2,7 +2,7 @@ import json import boto3 -from .utils import args_handler, data_files, DATA_PATH, INGESTION_STEP_MACHINE_ARN +from .utils import args_handler, data_files, DATA_PATH, DISCOVERY_TRIGGER_ARN items_path = os.path.join(DATA_PATH, "step_function_inputs") sf_client = boto3.client("stepfunctions") @@ -16,10 +16,14 @@ def insert_items(files): if type(events) != list: events = [events] for event in events: - response = sf_client.start_execution( - stateMachineArn=INGESTION_STEP_MACHINE_ARN, input=json.dumps(event) + lambda_client = boto3.client("lambda") + response = lambda_client.invoke( + FunctionName=DISCOVERY_TRIGGER_ARN, + InvocationType="Event", + Payload=json.dumps(event), ) - print(response) + + print(response) @args_handler diff --git a/scripts/utils.py b/scripts/utils.py index 2e51b2a4..815f7106 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -12,12 +12,13 @@ sts = boto3.client("sts") ACCOUNT_ID = sts.get_caller_identity().get("Account") -REGION = os.environ.get("AWS_REGION", "us-east-1") +REGION = os.environ.get("AWS_REGION", "us-west-2") APP_NAME = os.environ.get("APP_NAME") ENV = os.environ.get("ENV", "dev") SUBMIT_STAC_FUNCTION_NAME = f"{APP_NAME}-{ENV}-lambda-submit-stac-fn" -INGESTION_STEP_MACHINE_ARN = f"arn:aws:states:{REGION}:{ACCOUNT_ID}:stateMachine:{APP_NAME}-{ENV}-stepfunction-discover" +INGESTION_STEP_MACHINE_ARN = f"arn:aws:states:{REGION}:{ACCOUNT_ID}:stateMachine:{APP_NAME}-{ENV}-stepfunction-discovery-queue" +DISCOVERY_TRIGGER_ARN = f"arn:aws:lambda:{REGION}:{ACCOUNT_ID}:function:{APP_NAME}-{ENV}-lambda-trigger-discover-fn" def arguments(): From 1a80ed0b5c2f485b1f765719b8f58fb7c682ba25 Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Thu, 10 Nov 2022 10:17:50 -0600 Subject: [PATCH 2/8] Reduce concurrency in step functions to avoid lambda limits --- deploy/cdk/step_function_stack.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/cdk/step_function_stack.py b/deploy/cdk/step_function_stack.py index fe7d2168..80ac66dc 100644 --- a/deploy/cdk/step_function_stack.py +++ b/deploy/cdk/step_function_stack.py @@ -99,7 +99,7 @@ def _discovery_workflow( stepfunctions.Map( self, "Run concurrent queueing to cogify queue", - max_concurrency=3, + max_concurrency=1, items_path=stepfunctions.JsonPath.string_at("$.Payload.objects"), result_path=stepfunctions.JsonPath.DISCARD, output_path="$.Payload", @@ -111,7 +111,7 @@ def _discovery_workflow( stepfunctions.Map( self, "Run concurrent queueing to stac ready queue", - max_concurrency=3, + max_concurrency=1, items_path=stepfunctions.JsonPath.string_at("$.Payload.objects"), result_path=stepfunctions.JsonPath.DISCARD, output_path="$.Payload", @@ -166,7 +166,7 @@ def _cogify_workflow( cogify_workflow = stepfunctions.Map( self, "Run concurrent cogifications", - max_concurrency=3, + max_concurrency=1, items_path=stepfunctions.JsonPath.string_at("$"), ).iterator(cogify_task.next(enqueue_task)) @@ -209,7 +209,7 @@ def _publication_workflow( build_and_submit_stac_items = stepfunctions.Map( self, "Submit to STAC Ingestor", - max_concurrency=3, + max_concurrency=1, items_path=stepfunctions.JsonPath.string_at("$"), ).iterator(build_stac_item_task.next(submit_stac_item_task)) From 8854ce98b92ea558ac88a35d46c40db2360c3f9a Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Thu, 10 Nov 2022 12:06:16 -0600 Subject: [PATCH 3/8] Organize cmip6 ingests, throw with useful error when no files found --- .../p10/nex-gddp-cmip6-monthly-p10-rsds.json | 2 +- .../nex-gddp-cmip6-monthly-p10-sfcwind.json | 2 +- .../p50/nex-gddp-cmip6-monthly-p50-rsds.json | 2 +- .../p90/nex-gddp-cmip6-monthly-p90-rsds.json | 2 +- .../nex-gddp-cmip6-crossover.json | 0 .../nex-gddp-cmip6-daily-historical.json | 108 +++++++++--------- .../nex-gddp-cmip6-daily-ssp245-gfdl.json | 0 .../nex-gddp-cmip6-daily-ssp245-giss.json | 0 .../nex-gddp-cmip6-daily-ssp585-gfdl.json | 0 .../nex-gddp-cmip6-daily-ssp585-giss.json | 0 .../nex-gddp-cmip6-monthly-p10.json | 2 +- .../nex-gddp-cmip6-monthly-p50.json | 2 +- .../nex-gddp-cmip6-monthly-p90.json | 2 +- lambdas/s3-discovery/handler.py | 2 + scripts/item.py | 14 ++- 15 files changed, 71 insertions(+), 67 deletions(-) rename data/step_function_inputs/{ => nex-gddp-cmip6}/nex-gddp-cmip6-crossover.json (100%) rename data/step_function_inputs/{ => nex-gddp-cmip6}/nex-gddp-cmip6-daily-historical.json (67%) rename data/step_function_inputs/{ => nex-gddp-cmip6}/nex-gddp-cmip6-daily-ssp245-gfdl.json (100%) rename data/step_function_inputs/{ => nex-gddp-cmip6}/nex-gddp-cmip6-daily-ssp245-giss.json (100%) rename data/step_function_inputs/{ => nex-gddp-cmip6}/nex-gddp-cmip6-daily-ssp585-gfdl.json (100%) rename data/step_function_inputs/{ => nex-gddp-cmip6}/nex-gddp-cmip6-daily-ssp585-giss.json (100%) rename data/step_function_inputs/{ => nex-gddp-cmip6}/nex-gddp-cmip6-monthly-p10.json (97%) rename data/step_function_inputs/{ => nex-gddp-cmip6}/nex-gddp-cmip6-monthly-p50.json (97%) rename data/step_function_inputs/{ => nex-gddp-cmip6}/nex-gddp-cmip6-monthly-p90.json (97%) diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-rsds.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-rsds.json index 27318ee7..f1e50457 100644 --- a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-rsds.json +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-rsds.json @@ -1,5 +1,5 @@ { - "id": "nex-gddp-cmip6-daily-rdsds", + "id": "nex-gddp-cmip6-daily-rsds", "type": "Collection", "links": [], "title": "CMIP6 historical and projected surface downwelling shortwave radiation", diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-sfcwind.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-sfcwind.json index e079ebed..b806e077 100644 --- a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-sfcwind.json +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p10/nex-gddp-cmip6-monthly-p10-sfcwind.json @@ -24,7 +24,7 @@ } }, "license": "MIT", - "description": "CMIP6 historical and projected near-surface wind speed", + "description": "CMIP6 historical and projected near-surface wind speed.", "stac_version": "1.0.0", "dashboard:is_periodic": true, "dashboard:time_density": "day", diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-rsds.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-rsds.json index 27318ee7..f1e50457 100644 --- a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-rsds.json +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p50/nex-gddp-cmip6-monthly-p50-rsds.json @@ -1,5 +1,5 @@ { - "id": "nex-gddp-cmip6-daily-rdsds", + "id": "nex-gddp-cmip6-daily-rsds", "type": "Collection", "links": [], "title": "CMIP6 historical and projected surface downwelling shortwave radiation", diff --git a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-rsds.json b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-rsds.json index 27318ee7..f1e50457 100644 --- a/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-rsds.json +++ b/data/collections/nex-gddp-cmip6/nex-gddp-cmip6-monthly/p90/nex-gddp-cmip6-monthly-p90-rsds.json @@ -1,5 +1,5 @@ { - "id": "nex-gddp-cmip6-daily-rdsds", + "id": "nex-gddp-cmip6-daily-rsds", "type": "Collection", "links": [], "title": "CMIP6 historical and projected surface downwelling shortwave radiation", diff --git a/data/step_function_inputs/nex-gddp-cmip6-crossover.json b/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-crossover.json similarity index 100% rename from data/step_function_inputs/nex-gddp-cmip6-crossover.json rename to data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-crossover.json diff --git a/data/step_function_inputs/nex-gddp-cmip6-daily-historical.json b/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-daily-historical.json similarity index 67% rename from data/step_function_inputs/nex-gddp-cmip6-daily-historical.json rename to data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-daily-historical.json index 4bb2e910..d4c3e21c 100644 --- a/data/step_function_inputs/nex-gddp-cmip6-daily-historical.json +++ b/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-daily-historical.json @@ -1,218 +1,218 @@ [ { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-hurs", "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/hurs/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GFDL-CM4", - "variable": "hurs" + "model": "GFDL-CM4" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-huss", "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/huss/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GFDL-CM4", - "variable": "huss" + "model": "GFDL-CM4" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-pr", "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/pr/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GFDL-CM4", - "variable": "pr" + "model": "GFDL-CM4" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-rlds", "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/rlds/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GFDL-CM4", - "variable": "rlds" + "model": "GFDL-CM4" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-rsds", "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/rsds/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GFDL-CM4", - "variable": "rsds" + "model": "GFDL-CM4" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-sfcwind", "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/sfcWind/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GFDL-CM4", - "variable": "sfcWind" + "model": "GFDL-CM4" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-tasmax", "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/tasmax/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GFDL-CM4", - "variable": "tasmax" + "model": "GFDL-CM4" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-tasmin", "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/tasmin/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GFDL-CM4", - "variable": "tasmin" + "model": "GFDL-CM4" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-tas", "prefix": "daily/GFDL-CM4/historical/r1i1p1f1/tas/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GFDL-CM4", - "variable": "tas" + "model": "GFDL-CM4" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-hurs", "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/hurs/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GISS-E2.1-G", - "variable": "hurs" + "model": "GISS-E2.1-G" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-huss", "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/huss/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GISS-E2.1-G", - "variable": "huss" + "model": "GISS-E2.1-G" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-pr", "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/pr/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GISS-E2.1-G", - "variable": "pr" + "model": "GISS-E2.1-G" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-rlds", "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/rlds/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GISS-E2.1-G", - "variable": "rlds" + "model": "GISS-E2.1-G" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-rsds", "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/rsds/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GISS-E2.1-G", - "variable": "rsds" + "model": "GISS-E2.1-G" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-sfcwind", "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/sfcWind/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GISS-E2.1-G", - "variable": "sfcWind" + "model": "GISS-E2.1-G" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-tasmax", "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/tasmax/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GISS-E2.1-G", - "variable": "tasmax" + "model": "GISS-E2.1-G" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-tasmin", "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/tasmin/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GISS-E2.1-G", - "variable": "tasmin" + "model": "GISS-E2.1-G" } }, { - "collection": "nex-gddp-cmip6-daily", + "collection": "nex-gddp-cmip6-daily-tas", "prefix": "daily/GISS-E2-1-G/historical/r1i1p1f1/tas/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", "discovery": "s3", + "datetime_range": "day", "properties": { "ssp": "historical", - "model": "GISS-E2.1-G", - "variable": "tas" + "model": "GISS-E2.1-G" } } ] \ No newline at end of file diff --git a/data/step_function_inputs/nex-gddp-cmip6-daily-ssp245-gfdl.json b/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-daily-ssp245-gfdl.json similarity index 100% rename from data/step_function_inputs/nex-gddp-cmip6-daily-ssp245-gfdl.json rename to data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-daily-ssp245-gfdl.json diff --git a/data/step_function_inputs/nex-gddp-cmip6-daily-ssp245-giss.json b/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-daily-ssp245-giss.json similarity index 100% rename from data/step_function_inputs/nex-gddp-cmip6-daily-ssp245-giss.json rename to data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-daily-ssp245-giss.json diff --git a/data/step_function_inputs/nex-gddp-cmip6-daily-ssp585-gfdl.json b/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-daily-ssp585-gfdl.json similarity index 100% rename from data/step_function_inputs/nex-gddp-cmip6-daily-ssp585-gfdl.json rename to data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-daily-ssp585-gfdl.json diff --git a/data/step_function_inputs/nex-gddp-cmip6-daily-ssp585-giss.json b/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-daily-ssp585-giss.json similarity index 100% rename from data/step_function_inputs/nex-gddp-cmip6-daily-ssp585-giss.json rename to data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-daily-ssp585-giss.json diff --git a/data/step_function_inputs/nex-gddp-cmip6-monthly-p10.json b/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-monthly-p10.json similarity index 97% rename from data/step_function_inputs/nex-gddp-cmip6-monthly-p10.json rename to data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-monthly-p10.json index 8873031c..127162e3 100644 --- a/data/step_function_inputs/nex-gddp-cmip6-monthly-p10.json +++ b/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-monthly-p10.json @@ -32,7 +32,7 @@ "datetime_range": "month" }, { - "collection": "nex-gddp-cmip6-monthly-p10-rds", + "collection": "nex-gddp-cmip6-monthly-p10-rsds", "prefix": "monthly/CMIP6_ensemble_p10/rsds/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", diff --git a/data/step_function_inputs/nex-gddp-cmip6-monthly-p50.json b/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-monthly-p50.json similarity index 97% rename from data/step_function_inputs/nex-gddp-cmip6-monthly-p50.json rename to data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-monthly-p50.json index ebaf9bb6..e0fdae75 100644 --- a/data/step_function_inputs/nex-gddp-cmip6-monthly-p50.json +++ b/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-monthly-p50.json @@ -32,7 +32,7 @@ "datetime_range": "month" }, { - "collection": "nex-gddp-cmip6-monthly-p50-rds", + "collection": "nex-gddp-cmip6-monthly-p50-rsds", "prefix": "monthly/CMIP6_ensemble_median/rsds/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", diff --git a/data/step_function_inputs/nex-gddp-cmip6-monthly-p90.json b/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-monthly-p90.json similarity index 97% rename from data/step_function_inputs/nex-gddp-cmip6-monthly-p90.json rename to data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-monthly-p90.json index 1c8d8f22..550ad931 100644 --- a/data/step_function_inputs/nex-gddp-cmip6-monthly-p90.json +++ b/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-monthly-p90.json @@ -32,7 +32,7 @@ "datetime_range": "month" }, { - "collection": "nex-gddp-cmip6-monthly-p90-rds", + "collection": "nex-gddp-cmip6-monthly-p90-rsds", "prefix": "monthly/CMIP6_ensemble_p90/rsds/", "bucket": "nex-gddp-cmip6-cog", "filename_regex": "^.*.tif", diff --git a/lambdas/s3-discovery/handler.py b/lambdas/s3-discovery/handler.py index 1b25a1b1..c8e96e8c 100644 --- a/lambdas/s3-discovery/handler.py +++ b/lambdas/s3-discovery/handler.py @@ -42,6 +42,8 @@ def handler(event, context): file_objs_size = 0 payload = {**event, "cogify": cogify, "objects": []} for page in pages: + if "Contents" not in page: + raise Exception(f"No files found at s3://{bucket}/{prefix}") for obj in page["Contents"]: # The limit is advertised at 256000, but we'll preserve some breathing room if file_objs_size > 230000: diff --git a/scripts/item.py b/scripts/item.py index 9b06e6c9..b172e69c 100644 --- a/scripts/item.py +++ b/scripts/item.py @@ -15,15 +15,17 @@ def insert_items(files): events = json.load(open(filename)) if type(events) != list: events = [events] + print(events) for event in events: + print(event) lambda_client = boto3.client("lambda") - response = lambda_client.invoke( - FunctionName=DISCOVERY_TRIGGER_ARN, - InvocationType="Event", - Payload=json.dumps(event), - ) + # response = lambda_client.invoke( + # FunctionName=DISCOVERY_TRIGGER_ARN, + # InvocationType="Event", + # Payload=json.dumps(event), + # ) - print(response) + # print(response) @args_handler From ab00adbba5c34446c52efe6beee24d99c9a4116c Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Thu, 10 Nov 2022 12:12:46 -0600 Subject: [PATCH 4/8] Print insertion response after every new job --- scripts/item.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/scripts/item.py b/scripts/item.py index b172e69c..13338957 100644 --- a/scripts/item.py +++ b/scripts/item.py @@ -15,17 +15,15 @@ def insert_items(files): events = json.load(open(filename)) if type(events) != list: events = [events] - print(events) for event in events: - print(event) lambda_client = boto3.client("lambda") - # response = lambda_client.invoke( - # FunctionName=DISCOVERY_TRIGGER_ARN, - # InvocationType="Event", - # Payload=json.dumps(event), - # ) + response = lambda_client.invoke( + FunctionName=DISCOVERY_TRIGGER_ARN, + InvocationType="Event", + Payload=json.dumps(event), + ) - # print(response) + print(response) @args_handler From 55cfa7f01f7fd89b4782ad389f4b0395e5b5c87a Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Thu, 10 Nov 2022 12:30:25 -0600 Subject: [PATCH 5/8] Retry all lambdas to avoid transient TooManyRequest errors --- deploy/cdk/step_function_stack.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/deploy/cdk/step_function_stack.py b/deploy/cdk/step_function_stack.py index 80ac66dc..adeb8bdf 100644 --- a/deploy/cdk/step_function_stack.py +++ b/deploy/cdk/step_function_stack.py @@ -73,11 +73,21 @@ def _discovery_workflow( lambda_stack.s3_discovery_lambda, ) + s3_discovery_task.add_retry( + interval=core.Duration.seconds(2), + max_attempts=5, + ) + cmr_discovery_task = self._lambda_task( "CMR Discover Task", lambda_stack.cmr_discovery_lambda, ) + cmr_discovery_task.add_retry( + interval=core.Duration.seconds(2), + max_attempts=5, + ) + enqueue_cogify_task = self._sqs_task( "Send to Cogify queue", queue=queue_stack.cogify_queue, From f02e625fe592a311f5fd948dee824bf52408fc38 Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Mon, 14 Nov 2022 11:48:33 -0600 Subject: [PATCH 6/8] Propagate through event parameters to file objects so date arguments are used --- .../nex-gddp-cmip6/nex-gddp-cmip6-crossover.json | 6 ++++-- lambdas/s3-discovery/handler.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-crossover.json b/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-crossover.json index bfda8fd5..c6d6ddc8 100644 --- a/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-crossover.json +++ b/data/step_function_inputs/nex-gddp-cmip6/nex-gddp-cmip6-crossover.json @@ -3,8 +3,9 @@ "collection": "nex-gddp-cmip6-crossover", "prefix": "crossing/", "bucket": "nex-gddp-cmip6-cog", - "filename_regex": "tas_CrossingYear_CMIP6_ssp245.tif", + "filename_regex": "^.*245.tif", "discovery": "s3", + "single_datetime": "2015-01-01T00:00:00Z", "properties": { "ssp": "245" } @@ -13,8 +14,9 @@ "collection": "nex-gddp-cmip6-crossover", "prefix": "crossing/", "bucket": "nex-gddp-cmip6-cog", - "filename_regex": "tas_CrossingYear_CMIP6_ssp585.tif", + "filename_regex": "^.*585.tif", "discovery": "s3", + "single_datetime": "2015-01-01T00:00:00Z", "properties": { "ssp": "585" } diff --git a/lambdas/s3-discovery/handler.py b/lambdas/s3-discovery/handler.py index c8e96e8c..1623ac59 100644 --- a/lambdas/s3-discovery/handler.py +++ b/lambdas/s3-discovery/handler.py @@ -41,6 +41,18 @@ def handler(event, context): file_objs_size = 0 payload = {**event, "cogify": cogify, "objects": []} + + # Propagate forward optional datetime arguments + date_fields = {} + if "single_datetime" in event: + date_fields["single_datetime"] = event["single_datetime"] + if "start_datetime" in event: + date_fields["start_datetime"] = event["start_datetime"] + if "end_datetime" in event: + date_fields["end_datetime"] = event["end_datetime"] + if "datetime_range" in event: + date_fields["datetime_range"] = event["datetime_range"] + for page in pages: if "Contents" not in page: raise Exception(f"No files found at s3://{bucket}/{prefix}") @@ -57,6 +69,7 @@ def handler(event, context): "s3_filename": f"s3://{bucket}/{filename}", "upload": event.get("upload", False), "properties": properties, + **date_fields } payload["objects"].append(file_obj) file_obj_size = len(json.dumps(file_obj, ensure_ascii=False).encode("utf8")) From b0ec0ac5a021f2ad28e2518ed726435af4e26263 Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Mon, 14 Nov 2022 12:48:04 -0600 Subject: [PATCH 7/8] Lint --- lambdas/s3-discovery/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/s3-discovery/handler.py b/lambdas/s3-discovery/handler.py index 1623ac59..a79361bb 100644 --- a/lambdas/s3-discovery/handler.py +++ b/lambdas/s3-discovery/handler.py @@ -69,7 +69,7 @@ def handler(event, context): "s3_filename": f"s3://{bucket}/{filename}", "upload": event.get("upload", False), "properties": properties, - **date_fields + **date_fields, } payload["objects"].append(file_obj) file_obj_size = len(json.dumps(file_obj, ensure_ascii=False).encode("utf8")) From ace5f5af51d1f2daa027988fed6eb9f69883560f Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Mon, 21 Nov 2022 11:58:44 -0600 Subject: [PATCH 8/8] Add 'day' to event validation --- lambdas/build-stac/utils/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/build-stac/utils/events.py b/lambdas/build-stac/utils/events.py index 486fe382..da1bf913 100644 --- a/lambdas/build-stac/utils/events.py +++ b/lambdas/build-stac/utils/events.py @@ -5,7 +5,7 @@ import pystac -INTERVAL = Literal["month", "year"] +INTERVAL = Literal["month", "year", "day"] class BaseEvent(BaseModel, frozen=True):