From 7213a91cb4bd6d6ed72e4cab33759f5030f59693 Mon Sep 17 00:00:00 2001 From: Kevin Morales Date: Sun, 1 Oct 2023 09:26:00 -0400 Subject: [PATCH 01/16] fix asf logger method name error --- asf_search/ASFSearchOptions/ASFSearchOptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asf_search/ASFSearchOptions/ASFSearchOptions.py b/asf_search/ASFSearchOptions/ASFSearchOptions.py index 5200b189..8b1103e7 100644 --- a/asf_search/ASFSearchOptions/ASFSearchOptions.py +++ b/asf_search/ASFSearchOptions/ASFSearchOptions.py @@ -114,7 +114,7 @@ def merge_args(self, **kwargs) -> None: # Spit out warning if the value is something other than the default: if not self._is_val_default(key): msg = f'While merging search options, existing option {key}:{getattr(self, key, None)} overwritten by kwarg with value {kwargs[key]}' - ASF_LOGGER.warging(msg) + ASF_LOGGER.warning(msg) warnings.warn(msg) self.__setattr__(key, kwargs[key]) From d721c22795a0824cdfc30626f764d45843a9f92b Mon Sep 17 00:00:00 2001 From: kim Date: Wed, 18 Oct 2023 15:36:08 -0800 Subject: [PATCH 02/16] adds ERS-1/2, JERS-1, and RADARASAT-1 to list of products that use FRAME_NUMBER umm field --- asf_search/CMR/translate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asf_search/CMR/translate.py b/asf_search/CMR/translate.py index 867a68e7..ab74d44a 100644 --- a/asf_search/CMR/translate.py +++ b/asf_search/CMR/translate.py @@ -199,7 +199,7 @@ def translate_product(item: dict) -> dict: if properties['platform'] is None: properties['platform'] = get(umm, 'Platforms', 0, 'ShortName') - asf_frame_platforms = ['Sentinel-1A', 'Sentinel-1B', 'ALOS', 'SENTINEL-1A', 'SENTINEL-1B'] + asf_frame_platforms = ['Sentinel-1A', 'Sentinel-1B', 'ALOS', 'SENTINEL-1A', 'SENTINEL-1B', 'ERS-1', 'ERS-2', 'JERS-1', 'RADARSAT-1'] if properties['platform'] in asf_frame_platforms: properties['frameNumber'] = cast(int, get(umm, 'AdditionalAttributes', ('Name', 'FRAME_NUMBER'), 'Values', 0)) else: From 93de226ed001e96072c107d6d02e7e59a50ad290 Mon Sep 17 00:00:00 2001 From: kim Date: Wed, 18 Oct 2023 15:38:24 -0800 Subject: [PATCH 03/16] update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd48bd8b..befca2d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,11 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - --> +------ +## [v6.6.4](https://github.com/asfadmin/Discovery-asf_search/compare/v6.6.3...v6.6.4) +### Fixed +- ERS-1, ERS-2, JERS-1, and RADARSAT-1 now assign `FRAME_NUMBER` to the `frameNumber` properties field + ------ ## [v6.6.3](https://github.com/asfadmin/Discovery-asf_search/compare/v6.6.2...v6.6.3) ### Fixed From c28eadc7700ca9ae24c9907cd7c385f64bf759a9 Mon Sep 17 00:00:00 2001 From: kim Date: Fri, 3 Nov 2023 13:43:02 -0800 Subject: [PATCH 04/16] adds datasets keyword, removes explcicit umm versioning, update changelog --- CHANGELOG.md | 6 + asf_search/ASFSearchOptions/validator_map.py | 11 +- asf_search/CMR/__init__.py | 3 +- asf_search/CMR/datasets.py | 363 +++++++++++++++++++ asf_search/CMR/subquery.py | 2 +- asf_search/CMR/translate.py | 10 + asf_search/constants/INTERNAL.py | 2 +- asf_search/search/search.py | 1 + asf_search/search/search_generator.py | 1 + 9 files changed, 391 insertions(+), 8 deletions(-) create mode 100644 asf_search/CMR/datasets.py diff --git a/CHANGELOG.md b/CHANGELOG.md index befca2d7..6156cf54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,12 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). --> ------ ## [v6.6.4](https://github.com/asfadmin/Discovery-asf_search/compare/v6.6.3...v6.6.4) +### Added +- Adds new `datasets` keyword to `search()` as an alternative to `platform`. Allows users to get results from multiple platforms at once in a single page. + +### Changed +- Changes `CMR_FORMAT_EXT` constant from `umm_json_v1_4` to `umm_json`, umm returned from CMR will now be in latest umm format by default + ### Fixed - ERS-1, ERS-2, JERS-1, and RADARSAT-1 now assign `FRAME_NUMBER` to the `frameNumber` properties field diff --git a/asf_search/ASFSearchOptions/validator_map.py b/asf_search/ASFSearchOptions/validator_map.py index 64162bc1..65972b4c 100644 --- a/asf_search/ASFSearchOptions/validator_map.py +++ b/asf_search/ASFSearchOptions/validator_map.py @@ -57,12 +57,13 @@ def validate(key, value): 'instrument': parse_string, 'collections': parse_string_list, 'temporalBaselineDays': parse_string_list, - + 'datasets': parse_string_list, + 'absoluteBurstID': parse_int_list, + 'relativeBurstID': parse_int_list, + 'fullBurstID': parse_string_list, + # Config parameters Parser 'session': parse_session, 'host': parse_string, - 'provider': parse_string, - 'absoluteBurstID': parse_int_list, - 'relativeBurstID': parse_int_list, - 'fullBurstID': parse_string_list + 'provider': parse_string } diff --git a/asf_search/CMR/__init__.py b/asf_search/CMR/__init__.py index f382ebda..9a9e9d10 100644 --- a/asf_search/CMR/__init__.py +++ b/asf_search/CMR/__init__.py @@ -1,4 +1,5 @@ from .MissionList import get_campaigns from .subquery import build_subqueries from .translate import translate_product, translate_opts, get_additional_fields -from .field_map import field_map \ No newline at end of file +from .field_map import field_map +from .datasets import platform_datasets diff --git a/asf_search/CMR/datasets.py b/asf_search/CMR/datasets.py new file mode 100644 index 00000000..8c58245d --- /dev/null +++ b/asf_search/CMR/datasets.py @@ -0,0 +1,363 @@ +platform_datasets = { + "SENTINEL-1": [ + "C1214470488-ASF", + "C1327985661-ASF", + "C1214470533-ASF", + "C1214470576-ASF", + "C1327985645-ASF", + "C1214470496-ASF", + "C1214470532-ASF", + "C1214472977-ASF", + "C1214472336-ASF", + "C1266376001-ASF", + "C1214470732-ASF", + "C1327985660-ASF", + "C1327985741-ASF", + "C1327985578-ASF", + "C1327985650-ASF", + "C1327985617-ASF", + "C1327985579-ASF", + "C1327985619-ASF", + "C1214472994-ASF", + "C1214473170-ASF", + "C1327985646-ASF", + "C1327985740-ASF", + "C1327985739-ASF", + "C1214470561-ASF", + "C1214471521-ASF", + "C1214470682-ASF", + "C1327985647-ASF", + "C1214471197-ASF", + "C1214471960-ASF", + "C1214472978-ASF", + "C1214473165-ASF", + "C1327985697-ASF", + "C1327985651-ASF", + "C1327985644-ASF", + "C1327985571-ASF", + "C1327985674-ASF", + "C1205428742-ASF", + "C1216244348-ASF", + "C1212201032-ASF", + "C1216244589-ASF", + "C1212212560-ASF", + "C1205264459-ASF", + "C1216244594-ASF", + "C1216244593-ASF", + "C1208117434-ASF", + "C1212209035-ASF", + "C1212209226-ASF", + "C1216244585-ASF", + "C1216244592-ASF", + "C1208115009-ASF", + "C1216244595-ASF", + "C1212158327-ASF", + "C1215704763-ASF", + "C1212158318-ASF", + "C1216244590-ASF", + "C1216244601-ASF", + "C1216244600-ASF", + "C1212212493-ASF", + "C1212158326-ASF", + "C1212233976-ASF", + "C1216244591-ASF", + "C1216244587-ASF", + "C1216244598-ASF", + "C1216244586-ASF", + "C1212200781-ASF", + "C1212209075-ASF", + "C1216244597-ASF", + "C1216244596-ASF", + "C1216244588-ASF", + "C1216244599-ASF", + "C1234413245-ASFDEV", + "C1234413263-ASFDEV", + "C1234413229-ASFDEV", + "C1234413247-ASFDEV", + "C1234413237-ASFDEV", + "C1234413238-ASFDEV", + "C1234413248-ASFDEV", + "C1234413255-ASFDEV", + "C1234413236-ASFDEV", + "C1234413230-ASFDEV", + "C1234413232-ASFDEV", + "C1234413254-ASFDEV", + "C1234413256-ASFDEV", + "C1234413235-ASFDEV", + "C1234413253-ASFDEV", + "C1234413240-ASFDEV", + "C1234413234-ASFDEV", + "C1234413241-ASFDEV", + "C1234413252-ASFDEV", + "C1234413250-ASFDEV", + "C1234413259-ASFDEV", + "C1234413233-ASFDEV", + "C1234413243-ASFDEV", + "C1234413244-ASFDEV", + "C1234413251-ASFDEV", + "C1234413261-ASFDEV", + "C1234413262-ASFDEV", + "C1234413258-ASFDEV", + "C1244552887-ASFDEV", + "C1234413228-ASFDEV", + "C1234413231-ASFDEV", + "C1234413239-ASFDEV", + "C1234413242-ASFDEV", + "C1234413246-ASFDEV", + "C1234413249-ASFDEV", + "C1234413257-ASFDEV", + "C1234413260-ASFDEV", + "C1257175154-ASFDEV", + "C1244598379-ASFDEV", + "C1240784657-ASFDEV", + ], + "OPERA-S1": [ + "C2777443834-ASF", + "C2777436413-ASF", + "C1258354200-ASF", # maybe extra? + "C1259982010-ASF", + "C1257995185-ASF", + "C1259976861-ASF", + "C1258354201-ASF", + "C1259981910-ASF", + "C1257995186-ASF", + "C1259974840-ASF", + ], + "SLC-BURST": ["C2709161906-ASF", "C1257024016-ASF"], + "ALOS PALSAR": [ + "C1206487504-ASF", + "C1206485940-ASF", + "C1206487217-ASF", + "C1206156901-ASF", + "C1206485320-ASF", + "C1206485527-ASF", + "C2011599335-ASF", + "C1207181535-ASF", + "C1207710476-ASF", + "C1239927797-ASF", + "C1205261223-ASF", + "C1208013295-ASF", + "C1207019609-ASF", + "C1229740239-ASF", + "C1239611505-ASFDEV", + "C1238733834-ASFDEV", + ], + "ALOS AVNIR-2": [ + "C1808440897-ASF", + "C1234712303-ASF", + "C1233629671-ASF", + "C1234413224-ASFDEV", + ], + "SIR-C": [ + "C1661710578-ASF", + "C1661710581-ASF", + "C1661710583-ASF", + "C1661710586-ASF", + "C1661710588-ASF", + "C1661710590-ASF", + "C1661710593-ASF", + "C1661710596-ASF", + "C1661710597-ASF", + "C1661710600-ASF", + "C1661710603-ASF", + "C1661710604-ASF", + "C1226557813-ASF", + "C1226557814-ASF", + "C1226557819-ASF", + "C1226557809-ASF", + "C1226557808-ASF", + "C1226557810-ASF", + "C1226557811-ASF", + "C1226557812-ASF", + "C1226557815-ASF", + "C1226557816-ASF", + "C1226557817-ASF", + "C1226557818-ASF", + "C1234413270-ASFDEV", + "C1234413271-ASFDEV", + "C1234413264-ASFDEV", + "C1234413265-ASFDEV", + "C1234413266-ASFDEV", + "C1234413267-ASFDEV", + "C1234413268-ASFDEV", + "C1234413269-ASFDEV", + "C1234413272-ASFDEV", + "C1234413273-ASFDEV", + "C1234413274-ASFDEV", + "C1234413275-ASFDEV", + ], + "ARIA S1 GUNW": [ + "C1595422627-ASF", + "C1596065640-ASF", + "C1596065639-ASF", + "C1596065641-ASF", + "C1595765183-ASF", + "C1225776654-ASF", + "C1225776655-ASF", + "C1225776657-ASF", + "C1225776658-ASF", + "C1225776659-ASF", + ], + "SMAP": [ + "C1243122884-ASF", + "C1243124139-ASF", + "C1214473171-ASF", + "C1243149604-ASF", + "C1214473426-ASF", + "C1243119801-ASF", + "C1214473839-ASF", + "C1243133204-ASF", + "C1243197402-ASF", + "C1243215430-ASF", + "C1243124754-ASF", + "C1243141638-ASF", + "C1243162394-ASF", + "C1243168733-ASF", + "C1243168866-ASF", + "C1214473308-ASF", + "C1243253631-ASF", + "C1243133445-ASF", + "C1214473550-ASF", + "C1243197502-ASF", + "C1243126328-ASF", + "C1214474243-ASF", + "C1243216659-ASF", + "C1243129847-ASF", + "C1214473367-ASF", + "C1243268956-ASF", + "C1243144528-ASF", + "C1214473624-ASF", + "C1243228612-ASF", + "C1243136142-ASF", + "C1214474435-ASF", + "C1243255360-ASF", + "C1243140611-ASF", + "C1233103964-ASF", + "C1216074923-ASF", + "C1212243761-ASF", + "C1213091807-ASF", + "C1216074922-ASF", + "C1212243437-ASF", + "C1213096699-ASF", + "C1216074750-ASF", + "C1212249653-ASF", + "C1213101573-ASF", + "C1216074751-ASF", + "C1213136240-ASF", + "C1216074755-ASF", + "C1213136752-ASF", + "C1213136799-ASF", + "C1213136709-ASF", + "C1213136844-ASF", + "C1212249811-ASF", + "C1213125007-ASF", + "C1216074919-ASF", + "C1212196951-ASF", + "C1213115690-ASF", + "C1216074758-ASF", + "C1212243666-ASF", + "C1213115896-ASF", + "C1216074761-ASF", + "C1212250364-ASF", + "C1213134622-ASF", + "C1216074770-ASF", + "C1212246173-ASF", + "C1213125156-ASF", + "C1216074764-ASF", + "C1212249773-ASF", + "C1213134486-ASF", + "C1233101609-ASF", + ], + "UAVSAR": [ + "C1214353986-ASF", + "C1214336717-ASF", + "C1214336045-ASF", + "C1214335430-ASF", + "C1214335471-ASF", + "C1214335903-ASF", + "C1214336154-ASF", + "C1214336554-ASF", + "C1214353593-ASF", + "C1214353754-ASF", + "C1214353859-ASF", + "C1214337770-ASF", + "C1214354144-ASF", + "C1214354235-ASF", + "C1214343609-ASF", + "C1214354031-ASF", + "C1214408428-ASF", + "C1214419355-ASF", + "C1210487703-ASF", + "C1212030772-ASF", + "C1206116665-ASF", + "C1206132445-ASF", + "C1211962154-ASF", + "C1212001698-ASF", + "C1212005594-ASF", + "C1212019993-ASF", + "C1207638502-ASF", + "C1210025872-ASF", + "C1210485039-ASF", + "C1207188317-ASF", + "C1210546638-ASF", + "C1206122195-ASF", + "C1209970710-ASF", + "C1207038647-ASF", + "C1210599503-ASF", + "C1210599673-ASF", + ], + "RADARSAT-1": [ + "C1206897141-ASF", + "C1206936391-ASF", + "C1205181982-ASF", + "C1215670813-ASF", + "C1215709884-ASF", + "C1215709880-ASF", + "C1215714443-ASF", + "C1215775284-ASF", + "C1215614037-ASF", + ], + "ERS": [ + "C1210197768-ASF", + "C1211627521-ASF", + "C1208794942-ASF", + "C1209373626-ASF", + "C1207143701-ASF", + "C1205261222-ASF", + "C1205302527-ASF", + "C1207144966-ASF", + ], + "JERS-1": [ + "C1208662092-ASF", + "C1207933168-ASF", + "C1207175327-ASF", + "C1207177736-ASF", + ], + "AIRSAR": [ + "C1213921661-ASF", + "C1213921626-ASF", + "C1213928843-ASF", + "C1213927035-ASF", + "C179001730-ASF", + "C1213925022-ASF", + "C1213926419-ASF", + "C1213926777-ASF", + "C1213927939-ASF", + "C1213928209-ASF", + "C1205256880-ASF", + "C1208652494-ASF", + "C1000000306-ASF", + "C1208713702-ASF", + "C1208707768-ASF", + "C1208655639-ASF", + "C1208680681-ASF", + "C1208691361-ASF", + "C1208703384-ASF", + ], + "SEASAT": [ + "C1206500826-ASF", + "C1206500991-ASF", + "C1206752770-ASF", + "C1206144699-ASF", + ], +} diff --git a/asf_search/CMR/subquery.py b/asf_search/CMR/subquery.py index f37a0ef0..8c00c789 100644 --- a/asf_search/CMR/subquery.py +++ b/asf_search/CMR/subquery.py @@ -22,7 +22,7 @@ def build_subqueries(opts: ASFSearchOptions) -> List[ASFSearchOptions]: if params.get('product_list') is not None: params['product_list'] = chunk_list(params['product_list'], CMR_PAGE_SIZE) - list_param_names = ['platform', 'season', 'collections'] # these parameters will dodge the subquery system + list_param_names = ['platform', 'season', 'collections', 'datasets'] # these parameters will dodge the subquery system skip_param_names = ['maxResults']# these params exist in opts, but shouldn't be passed on to subqueries at ALL params = dict([ (k, v) for k, v in params.items() if k not in skip_param_names ]) diff --git a/asf_search/CMR/translate.py b/asf_search/CMR/translate.py index ab74d44a..1eb91e51 100644 --- a/asf_search/CMR/translate.py +++ b/asf_search/CMR/translate.py @@ -7,6 +7,7 @@ from shapely.geometry import Polygon from shapely.geometry.base import BaseGeometry from .field_map import field_map +from .datasets import platform_datasets import logging @@ -46,7 +47,16 @@ def translate_opts(opts: ASFSearchOptions) -> list: # If you need to use the temporal key: if any(key in dict_opts for key in ['start', 'end', 'season']): dict_opts = fix_date(dict_opts) + + if 'datasets' in dict_opts: + if 'collections' not in dict_opts: + dict_opts['collections'] = [] + + for collection in dict_opts['datasets']: + dict_opts['collections'].extend(platform_datasets[collection]) + dict_opts.pop('datasets') + # convert the above parameters to a list of key/value tuples cmr_opts = [] for (key, val) in dict_opts.items(): diff --git a/asf_search/constants/INTERNAL.py b/asf_search/constants/INTERNAL.py index 8ef3e144..124999dc 100644 --- a/asf_search/constants/INTERNAL.py +++ b/asf_search/constants/INTERNAL.py @@ -1,7 +1,7 @@ ASF_AUTH_HOST = 'auth.asf.alaska.edu' CMR_HOST = 'cmr.earthdata.nasa.gov' -CMR_FORMAT_EXT = 'umm_json_v1_4' +CMR_FORMAT_EXT = 'umm_json' CMR_GRANULE_PATH = f'/search/granules.{CMR_FORMAT_EXT}' CMR_COLLECTIONS = f'/search/collections' CMR_COLLECTIONS_PATH = f'{CMR_COLLECTIONS}.{CMR_FORMAT_EXT}' diff --git a/asf_search/search/search.py b/asf_search/search/search.py index bf7b0529..8603038a 100644 --- a/asf_search/search/search.py +++ b/asf_search/search/search.py @@ -40,6 +40,7 @@ def search( fullBurstID: Union[str, Iterable[str]] = None, collections: Union[str, Iterable[str]] = None, temporalBaselineDays: Union[str, Iterable[str]] = None, + datasets: Union[str, Iterable[str]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> ASFSearchResults: diff --git a/asf_search/search/search_generator.py b/asf_search/search/search_generator.py index 8763ea7e..4ddcde1e 100644 --- a/asf_search/search/search_generator.py +++ b/asf_search/search/search_generator.py @@ -54,6 +54,7 @@ def search_generator( fullBurstID: Union[str, Iterable[str]] = None, collections: Union[str, Iterable[str]] = None, temporalBaselineDays: Union[str, Iterable[str]] = None, + datasets: Union[str, Iterable[str]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> Generator[ASFSearchResults, None, None]: From aa5537ccdf96b7ee7ac5ff3b55d9325c23383297 Mon Sep 17 00:00:00 2001 From: kim Date: Fri, 3 Nov 2023 14:19:15 -0800 Subject: [PATCH 05/16] adds RTC/CSLC-STATIC concept ids to opera dataset collections --- asf_search/CMR/datasets.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/asf_search/CMR/datasets.py b/asf_search/CMR/datasets.py index 8c58245d..e1cb8c53 100644 --- a/asf_search/CMR/datasets.py +++ b/asf_search/CMR/datasets.py @@ -114,6 +114,8 @@ "OPERA-S1": [ "C2777443834-ASF", "C2777436413-ASF", + "C2795135174-ASF", # RTC-STATIC + "C2795135668-ASF", # CSLC-STATIC "C1258354200-ASF", # maybe extra? "C1259982010-ASF", "C1257995185-ASF", @@ -122,6 +124,7 @@ "C1259981910-ASF", "C1257995186-ASF", "C1259974840-ASF", + ], "SLC-BURST": ["C2709161906-ASF", "C1257024016-ASF"], "ALOS PALSAR": [ From d33cdbcf62802202e662cea1c9a2dc691da6f7b4 Mon Sep 17 00:00:00 2001 From: kim Date: Mon, 6 Nov 2023 13:51:32 -0900 Subject: [PATCH 06/16] error handling for invalid dataset passed to datasets keyword --- asf_search/CMR/translate.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/asf_search/CMR/translate.py b/asf_search/CMR/translate.py index 1eb91e51..703bbfda 100644 --- a/asf_search/CMR/translate.py +++ b/asf_search/CMR/translate.py @@ -52,8 +52,11 @@ def translate_opts(opts: ASFSearchOptions) -> list: if 'collections' not in dict_opts: dict_opts['collections'] = [] - for collection in dict_opts['datasets']: - dict_opts['collections'].extend(platform_datasets[collection]) + for dataset in dict_opts['datasets']: + if dataset_collections := platform_datasets.get(dataset): + dict_opts['collections'].extend(dataset_collections) + else: + raise ValueError(f'Could not find dataset named "{dataset}" provided for datasets keyword.') dict_opts.pop('datasets') From cf9c311a6eb25aa3dadc721b8b5a74a0b0f45a3d Mon Sep 17 00:00:00 2001 From: kim Date: Mon, 6 Nov 2023 16:35:31 -0900 Subject: [PATCH 07/16] update OPERA-S1 dataset collections list --- asf_search/CMR/datasets.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/asf_search/CMR/datasets.py b/asf_search/CMR/datasets.py index e1cb8c53..342e48e3 100644 --- a/asf_search/CMR/datasets.py +++ b/asf_search/CMR/datasets.py @@ -124,6 +124,10 @@ "C1259981910-ASF", "C1257995186-ASF", "C1259974840-ASF", + "C1260726384-ASF", + "C1260721945-ASF", + "C1260721853-ASF", + "C1260726378-ASF", ], "SLC-BURST": ["C2709161906-ASF", "C1257024016-ASF"], From 19fd58779e22154d8d13ec885752448b896bed90 Mon Sep 17 00:00:00 2001 From: kim Date: Mon, 6 Nov 2023 17:00:09 -0900 Subject: [PATCH 08/16] WIP: adds test case for datasets keyword --- tests/Search/test_search.py | 42 ++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/Search/test_search.py b/tests/Search/test_search.py index e3d38c80..718c6b13 100644 --- a/tests/Search/test_search.py +++ b/tests/Search/test_search.py @@ -1,10 +1,13 @@ from numbers import Number from asf_search import ASFSearchOptions from asf_search.ASFProduct import ASFProduct +from asf_search.CMR.translate import get from asf_search.constants import INTERNAL from asf_search.exceptions import ASFSearchError from asf_search.search import search from asf_search.ASFSearchResults import ASFSearchResults +from asf_search.CMR import platform_datasets +from typing import List import requests @@ -75,4 +78,41 @@ def custom_matcher(request: requests.Request): with raises(ASFSearchError): results.raise_if_incomplete() - +def run_test_datasets_search(datasets: List): + should_raise_error = len([dataset for dataset in datasets if not platform_datasets.get(dataset)]) > 0 + + if should_raise_error: + with raises(ValueError): + search(datasets=datasets, maxResults=1) + + # WIP + # get collection concept-ids from shortName cmr query + else: + valid_collections = [] + for dataset in datasets: + valid_collections.extend(platform_datasets.get(dataset)) + + response = search(datasets=datasets, maxResults=250) + + retries = 3 + + while(retries != 0): + try: + # Granules don't keep track of their concept-ids, + # so query CMR to get them using each product's collection shortName + shortNames = '&'.join(list(set([f"shortName[]={get(product.umm, 'CollectionReference', 'ShortName')}" for product in response]))) + r = requests.get(f"https://cmr.earthdata.nasa.gov/search/collections.umm_json?{shortNames}&provider=ASF") + r.raise_for_status() + + items = r.json()['items'] + concept_ids = [item['meta']['concept-id'] for item in items] + + # check that results are limited to the expected datasets + for concept_id in concept_ids: + assert concept_id in valid_collections + + return + except requests.HTTPError: + retries -= 1 + + From ea4774cab754ee08ec522d7caeded5665606316a Mon Sep 17 00:00:00 2001 From: kim Date: Tue, 7 Nov 2023 10:39:51 -0900 Subject: [PATCH 09/16] adds test for datasets based search --- tests/Search/test_search.py | 48 ++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/tests/Search/test_search.py b/tests/Search/test_search.py index 718c6b13..5b9f3989 100644 --- a/tests/Search/test_search.py +++ b/tests/Search/test_search.py @@ -7,9 +7,9 @@ from asf_search.search import search from asf_search.ASFSearchResults import ASFSearchResults from asf_search.CMR import platform_datasets -from typing import List import requests +from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_exception_type from pytest import raises @@ -78,14 +78,20 @@ def custom_matcher(request: requests.Request): with raises(ASFSearchError): results.raise_if_incomplete() -def run_test_datasets_search(datasets: List): +@retry( + reraise=True, + wait=wait_fixed(3), + stop=stop_after_attempt(3), + retry=retry_if_exception_type((requests.ReadTimeout, requests.HTTPError)) +) +def test_datasets_search(): + datasets = ['SENTINEL-1', 'RADARSAT-1', 'UAVSAR'] should_raise_error = len([dataset for dataset in datasets if not platform_datasets.get(dataset)]) > 0 if should_raise_error: with raises(ValueError): search(datasets=datasets, maxResults=1) - # WIP # get collection concept-ids from shortName cmr query else: valid_collections = [] @@ -94,25 +100,17 @@ def run_test_datasets_search(datasets: List): response = search(datasets=datasets, maxResults=250) - retries = 3 - - while(retries != 0): - try: - # Granules don't keep track of their concept-ids, - # so query CMR to get them using each product's collection shortName - shortNames = '&'.join(list(set([f"shortName[]={get(product.umm, 'CollectionReference', 'ShortName')}" for product in response]))) - r = requests.get(f"https://cmr.earthdata.nasa.gov/search/collections.umm_json?{shortNames}&provider=ASF") - r.raise_for_status() - - items = r.json()['items'] - concept_ids = [item['meta']['concept-id'] for item in items] - - # check that results are limited to the expected datasets - for concept_id in concept_ids: - assert concept_id in valid_collections - - return - except requests.HTTPError: - retries -= 1 - - + # Granules don't keep track of their collection concept-ids, + # but they do have the collection names we can use to find them! + shortNames = '&'.join(list(set([f"shortName[]={get(product.umm, 'CollectionReference', 'ShortName')}" for product in response]))) + + r = requests.get(f"https://cmr.earthdata.nasa.gov/search/collections.umm_json?{shortNames}&provider=ASF") + r.raise_for_status() + + items = r.json()['items'] + concept_ids = [item['meta']['concept-id'] for item in items] + + # check that results are limited to the expected datasets + for concept_id in concept_ids: + assert concept_id in valid_collections + \ No newline at end of file From e80946adb9dc239be3ee2d335851ebce0330b3d7 Mon Sep 17 00:00:00 2001 From: kim Date: Tue, 7 Nov 2023 13:26:30 -0900 Subject: [PATCH 10/16] collects dataset concept ids by shortNames --- asf_search/CMR/__init__.py | 2 +- asf_search/CMR/datasets.py | 695 +++++++++++++++++------------------- asf_search/CMR/translate.py | 7 +- tests/Search/test_search.py | 41 +-- 4 files changed, 342 insertions(+), 403 deletions(-) diff --git a/asf_search/CMR/__init__.py b/asf_search/CMR/__init__.py index 9a9e9d10..905e8eb3 100644 --- a/asf_search/CMR/__init__.py +++ b/asf_search/CMR/__init__.py @@ -2,4 +2,4 @@ from .subquery import build_subqueries from .translate import translate_product, translate_opts, get_additional_fields from .field_map import field_map -from .datasets import platform_datasets +from .datasets import dataset_collections \ No newline at end of file diff --git a/asf_search/CMR/datasets.py b/asf_search/CMR/datasets.py index 342e48e3..b9bf61b6 100644 --- a/asf_search/CMR/datasets.py +++ b/asf_search/CMR/datasets.py @@ -1,370 +1,327 @@ -platform_datasets = { - "SENTINEL-1": [ - "C1214470488-ASF", - "C1327985661-ASF", - "C1214470533-ASF", - "C1214470576-ASF", - "C1327985645-ASF", - "C1214470496-ASF", - "C1214470532-ASF", - "C1214472977-ASF", - "C1214472336-ASF", - "C1266376001-ASF", - "C1214470732-ASF", - "C1327985660-ASF", - "C1327985741-ASF", - "C1327985578-ASF", - "C1327985650-ASF", - "C1327985617-ASF", - "C1327985579-ASF", - "C1327985619-ASF", - "C1214472994-ASF", - "C1214473170-ASF", - "C1327985646-ASF", - "C1327985740-ASF", - "C1327985739-ASF", - "C1214470561-ASF", - "C1214471521-ASF", - "C1214470682-ASF", - "C1327985647-ASF", - "C1214471197-ASF", - "C1214471960-ASF", - "C1214472978-ASF", - "C1214473165-ASF", - "C1327985697-ASF", - "C1327985651-ASF", - "C1327985644-ASF", - "C1327985571-ASF", - "C1327985674-ASF", - "C1205428742-ASF", - "C1216244348-ASF", - "C1212201032-ASF", - "C1216244589-ASF", - "C1212212560-ASF", - "C1205264459-ASF", - "C1216244594-ASF", - "C1216244593-ASF", - "C1208117434-ASF", - "C1212209035-ASF", - "C1212209226-ASF", - "C1216244585-ASF", - "C1216244592-ASF", - "C1208115009-ASF", - "C1216244595-ASF", - "C1212158327-ASF", - "C1215704763-ASF", - "C1212158318-ASF", - "C1216244590-ASF", - "C1216244601-ASF", - "C1216244600-ASF", - "C1212212493-ASF", - "C1212158326-ASF", - "C1212233976-ASF", - "C1216244591-ASF", - "C1216244587-ASF", - "C1216244598-ASF", - "C1216244586-ASF", - "C1212200781-ASF", - "C1212209075-ASF", - "C1216244597-ASF", - "C1216244596-ASF", - "C1216244588-ASF", - "C1216244599-ASF", - "C1234413245-ASFDEV", - "C1234413263-ASFDEV", - "C1234413229-ASFDEV", - "C1234413247-ASFDEV", - "C1234413237-ASFDEV", - "C1234413238-ASFDEV", - "C1234413248-ASFDEV", - "C1234413255-ASFDEV", - "C1234413236-ASFDEV", - "C1234413230-ASFDEV", - "C1234413232-ASFDEV", - "C1234413254-ASFDEV", - "C1234413256-ASFDEV", - "C1234413235-ASFDEV", - "C1234413253-ASFDEV", - "C1234413240-ASFDEV", - "C1234413234-ASFDEV", - "C1234413241-ASFDEV", - "C1234413252-ASFDEV", - "C1234413250-ASFDEV", - "C1234413259-ASFDEV", - "C1234413233-ASFDEV", - "C1234413243-ASFDEV", - "C1234413244-ASFDEV", - "C1234413251-ASFDEV", - "C1234413261-ASFDEV", - "C1234413262-ASFDEV", - "C1234413258-ASFDEV", - "C1244552887-ASFDEV", - "C1234413228-ASFDEV", - "C1234413231-ASFDEV", - "C1234413239-ASFDEV", - "C1234413242-ASFDEV", - "C1234413246-ASFDEV", - "C1234413249-ASFDEV", - "C1234413257-ASFDEV", - "C1234413260-ASFDEV", - "C1257175154-ASFDEV", - "C1244598379-ASFDEV", - "C1240784657-ASFDEV", - ], - "OPERA-S1": [ - "C2777443834-ASF", - "C2777436413-ASF", - "C2795135174-ASF", # RTC-STATIC - "C2795135668-ASF", # CSLC-STATIC - "C1258354200-ASF", # maybe extra? - "C1259982010-ASF", - "C1257995185-ASF", - "C1259976861-ASF", - "C1258354201-ASF", - "C1259981910-ASF", - "C1257995186-ASF", - "C1259974840-ASF", - "C1260726384-ASF", - "C1260721945-ASF", - "C1260721853-ASF", - "C1260726378-ASF", - - ], - "SLC-BURST": ["C2709161906-ASF", "C1257024016-ASF"], - "ALOS PALSAR": [ - "C1206487504-ASF", - "C1206485940-ASF", - "C1206487217-ASF", - "C1206156901-ASF", - "C1206485320-ASF", - "C1206485527-ASF", - "C2011599335-ASF", - "C1207181535-ASF", - "C1207710476-ASF", - "C1239927797-ASF", - "C1205261223-ASF", - "C1208013295-ASF", - "C1207019609-ASF", - "C1229740239-ASF", - "C1239611505-ASFDEV", - "C1238733834-ASFDEV", - ], - "ALOS AVNIR-2": [ - "C1808440897-ASF", - "C1234712303-ASF", - "C1233629671-ASF", - "C1234413224-ASFDEV", - ], - "SIR-C": [ - "C1661710578-ASF", - "C1661710581-ASF", - "C1661710583-ASF", - "C1661710586-ASF", - "C1661710588-ASF", - "C1661710590-ASF", - "C1661710593-ASF", - "C1661710596-ASF", - "C1661710597-ASF", - "C1661710600-ASF", - "C1661710603-ASF", - "C1661710604-ASF", - "C1226557813-ASF", - "C1226557814-ASF", - "C1226557819-ASF", - "C1226557809-ASF", - "C1226557808-ASF", - "C1226557810-ASF", - "C1226557811-ASF", - "C1226557812-ASF", - "C1226557815-ASF", - "C1226557816-ASF", - "C1226557817-ASF", - "C1226557818-ASF", - "C1234413270-ASFDEV", - "C1234413271-ASFDEV", - "C1234413264-ASFDEV", - "C1234413265-ASFDEV", - "C1234413266-ASFDEV", - "C1234413267-ASFDEV", - "C1234413268-ASFDEV", - "C1234413269-ASFDEV", - "C1234413272-ASFDEV", - "C1234413273-ASFDEV", - "C1234413274-ASFDEV", - "C1234413275-ASFDEV", - ], - "ARIA S1 GUNW": [ - "C1595422627-ASF", - "C1596065640-ASF", - "C1596065639-ASF", - "C1596065641-ASF", - "C1595765183-ASF", - "C1225776654-ASF", - "C1225776655-ASF", - "C1225776657-ASF", - "C1225776658-ASF", - "C1225776659-ASF", - ], - "SMAP": [ - "C1243122884-ASF", - "C1243124139-ASF", - "C1214473171-ASF", - "C1243149604-ASF", - "C1214473426-ASF", - "C1243119801-ASF", - "C1214473839-ASF", - "C1243133204-ASF", - "C1243197402-ASF", - "C1243215430-ASF", - "C1243124754-ASF", - "C1243141638-ASF", - "C1243162394-ASF", - "C1243168733-ASF", - "C1243168866-ASF", - "C1214473308-ASF", - "C1243253631-ASF", - "C1243133445-ASF", - "C1214473550-ASF", - "C1243197502-ASF", - "C1243126328-ASF", - "C1214474243-ASF", - "C1243216659-ASF", - "C1243129847-ASF", - "C1214473367-ASF", - "C1243268956-ASF", - "C1243144528-ASF", - "C1214473624-ASF", - "C1243228612-ASF", - "C1243136142-ASF", - "C1214474435-ASF", - "C1243255360-ASF", - "C1243140611-ASF", - "C1233103964-ASF", - "C1216074923-ASF", - "C1212243761-ASF", - "C1213091807-ASF", - "C1216074922-ASF", - "C1212243437-ASF", - "C1213096699-ASF", - "C1216074750-ASF", - "C1212249653-ASF", - "C1213101573-ASF", - "C1216074751-ASF", - "C1213136240-ASF", - "C1216074755-ASF", - "C1213136752-ASF", - "C1213136799-ASF", - "C1213136709-ASF", - "C1213136844-ASF", - "C1212249811-ASF", - "C1213125007-ASF", - "C1216074919-ASF", - "C1212196951-ASF", - "C1213115690-ASF", - "C1216074758-ASF", - "C1212243666-ASF", - "C1213115896-ASF", - "C1216074761-ASF", - "C1212250364-ASF", - "C1213134622-ASF", - "C1216074770-ASF", - "C1212246173-ASF", - "C1213125156-ASF", - "C1216074764-ASF", - "C1212249773-ASF", - "C1213134486-ASF", - "C1233101609-ASF", - ], - "UAVSAR": [ - "C1214353986-ASF", - "C1214336717-ASF", - "C1214336045-ASF", - "C1214335430-ASF", - "C1214335471-ASF", - "C1214335903-ASF", - "C1214336154-ASF", - "C1214336554-ASF", - "C1214353593-ASF", - "C1214353754-ASF", - "C1214353859-ASF", - "C1214337770-ASF", - "C1214354144-ASF", - "C1214354235-ASF", - "C1214343609-ASF", - "C1214354031-ASF", - "C1214408428-ASF", - "C1214419355-ASF", - "C1210487703-ASF", - "C1212030772-ASF", - "C1206116665-ASF", - "C1206132445-ASF", - "C1211962154-ASF", - "C1212001698-ASF", - "C1212005594-ASF", - "C1212019993-ASF", - "C1207638502-ASF", - "C1210025872-ASF", - "C1210485039-ASF", - "C1207188317-ASF", - "C1210546638-ASF", - "C1206122195-ASF", - "C1209970710-ASF", - "C1207038647-ASF", - "C1210599503-ASF", - "C1210599673-ASF", - ], - "RADARSAT-1": [ - "C1206897141-ASF", - "C1206936391-ASF", - "C1205181982-ASF", - "C1215670813-ASF", - "C1215709884-ASF", - "C1215709880-ASF", - "C1215714443-ASF", - "C1215775284-ASF", - "C1215614037-ASF", - ], - "ERS": [ - "C1210197768-ASF", - "C1211627521-ASF", - "C1208794942-ASF", - "C1209373626-ASF", - "C1207143701-ASF", - "C1205261222-ASF", - "C1205302527-ASF", - "C1207144966-ASF", - ], - "JERS-1": [ - "C1208662092-ASF", - "C1207933168-ASF", - "C1207175327-ASF", - "C1207177736-ASF", - ], - "AIRSAR": [ - "C1213921661-ASF", - "C1213921626-ASF", - "C1213928843-ASF", - "C1213927035-ASF", - "C179001730-ASF", - "C1213925022-ASF", - "C1213926419-ASF", - "C1213926777-ASF", - "C1213927939-ASF", - "C1213928209-ASF", - "C1205256880-ASF", - "C1208652494-ASF", - "C1000000306-ASF", - "C1208713702-ASF", - "C1208707768-ASF", - "C1208655639-ASF", - "C1208680681-ASF", - "C1208691361-ASF", - "C1208703384-ASF", - ], - "SEASAT": [ - "C1206500826-ASF", - "C1206500991-ASF", - "C1206752770-ASF", - "C1206144699-ASF", - ], +dataset_collections = { + "SENTINEL-1": { + "SENTINEL-1A_SLC": ["C1214470488-ASF", "C1205428742-ASF", "C1234413245-ASFDEV"], + "SENTINEL-1B_SLC": ["C1327985661-ASF", "C1216244348-ASF", "C1234413263-ASFDEV"], + "SENTINEL-1A_DP_GRD_HIGH": [ + "C1214470533-ASF", + "C1212201032-ASF", + "C1234413229-ASFDEV", + ], + "SENTINEL-1A_DP_META_GRD_HIGH": [ + "C1214470576-ASF", + "C1212209226-ASF", + "C1234413232-ASFDEV", + ], + "SENTINEL-1B_DP_GRD_HIGH": [ + "C1327985645-ASF", + "C1216244589-ASF", + "C1234413247-ASFDEV", + ], + "SENTINEL-1A_META_SLC": [ + "C1214470496-ASF", + "C1208117434-ASF", + "C1234413236-ASFDEV", + ], + "SENTINEL-1A_META_RAW": [ + "C1214470532-ASF", + "C1208115009-ASF", + "C1234413235-ASFDEV", + ], + "SENTINEL-1A_OCN": ["C1214472977-ASF", "C1212212560-ASF", "C1234413237-ASFDEV"], + "SENTINEL-1A_DP_META_GRD_MEDIUM": [ + "C1214472336-ASF", + "C1212212493-ASF", + "C1234413233-ASFDEV", + ], + "SENTINEL-1A_META_OCN": [ + "C1266376001-ASF", + "C1215704763-ASF", + "C1234413234-ASFDEV", + ], + "SENTINEL-1A_SP_META_GRD_HIGH": [ + "C1214470732-ASF", + "C1212158326-ASF", + "C1234413243-ASFDEV", + ], + "SENTINEL-1B_DP_GRD_MEDIUM": [ + "C1327985660-ASF", + "C1216244594-ASF", + "C1234413248-ASFDEV", + ], + "SENTINEL-1B_DP_META_GRD_HIGH": [ + "C1327985741-ASF", + "C1216244601-ASF", + "C1234413250-ASFDEV", + ], + "SENTINEL-1B_DP_META_GRD_MEDIUM": [ + "C1327985578-ASF", + "C1216244591-ASF", + "C1234413251-ASFDEV", + ], + "SENTINEL-1B_META_RAW": [ + "C1327985650-ASF", + "C1216244595-ASF", + "C1234413253-ASFDEV", + ], + "SENTINEL-1B_META_SLC": [ + "C1327985617-ASF", + "C1216244585-ASF", + "C1234413254-ASFDEV", + ], + "SENTINEL-1B_OCN": ["C1327985579-ASF", "C1216244593-ASF", "C1234413255-ASFDEV"], + "SENTINEL-1B_SP_META_GRD_HIGH": [ + "C1327985619-ASF", + "C1216244587-ASF", + "C1234413261-ASFDEV", + ], + "SENTINEL-1A_SP_GRD_MEDIUM": [ + "C1214472994-ASF", + "C1212158318-ASF", + "C1234413241-ASFDEV", + ], + "SENTINEL-1A_SP_META_GRD_MEDIUM": [ + "C1214473170-ASF", + "C1212233976-ASF", + "C1234413244-ASFDEV", + ], + "SENTINEL-1B_META_OCN": [ + "C1327985646-ASF", + "C1216244590-ASF", + "C1234413252-ASFDEV", + ], + "SENTINEL-1B_SP_GRD_MEDIUM": [ + "C1327985740-ASF", + "C1216244600-ASF", + "C1234413259-ASFDEV", + ], + "SENTINEL-1B_SP_META_GRD_MEDIUM": [ + "C1327985739-ASF", + "C1216244598-ASF", + "C1234413262-ASFDEV", + ], + "SENTINEL-1A_RAW": ["C1214470561-ASF", "C1205264459-ASF", "C1234413238-ASFDEV"], + "SENTINEL-1A_DP_GRD_MEDIUM": [ + "C1214471521-ASF", + "C1212209035-ASF", + "C1234413230-ASFDEV", + ], + "SENTINEL-1A_SP_GRD_HIGH": [ + "C1214470682-ASF", + "C1212158327-ASF", + "C1234413240-ASFDEV", + ], + "SENTINEL-1B_RAW": ["C1327985647-ASF", "C1216244592-ASF", "C1234413256-ASFDEV"], + "SENTINEL-1A_DP_GRD_FULL": [ + "C1214471197-ASF", + "C1212200781-ASF", + "C1234413228-ASFDEV", + ], + "SENTINEL-1A_DP_META_GRD_FULL": [ + "C1214471960-ASF", + "C1212209075-ASF", + "C1234413231-ASFDEV", + ], + "SENTINEL-1A_SP_GRD_FULL": ["C1214472978-ASF", "C1234413239-ASFDEV"], + "SENTINEL-1A_SP_META_GRD_FULL": ["C1214473165-ASF", "C1234413242-ASFDEV"], + "SENTINEL-1B_DP_GRD_FULL": [ + "C1327985697-ASF", + "C1216244597-ASF", + "C1234413246-ASFDEV", + ], + "SENTINEL-1B_DP_META_GRD_FULL": [ + "C1327985651-ASF", + "C1216244596-ASF", + "C1234413249-ASFDEV", + ], + "SENTINEL-1B_SP_GRD_FULL": [ + "C1327985644-ASF", + "C1216244588-ASF", + "C1234413257-ASFDEV", + ], + "SENTINEL-1B_SP_GRD_HIGH": [ + "C1327985571-ASF", + "C1216244586-ASF", + "C1234413258-ASFDEV", + ], + "SENTINEL-1B_SP_META_GRD_FULL": [ + "C1327985674-ASF", + "C1216244599-ASF", + "C1234413260-ASFDEV", + ], + "S1_Bursts": ["C1244552887-ASFDEV"], + "SENTINEL-1_BURSTS_DEV10": ["C1257175154-ASFDEV"], + "Sentinel-1_Burst_Map": ["C1244598379-ASFDEV"], + "Various Browse Images": ["C1240784657-ASFDEV"], + }, + "OPERA-S1": { + "OPERA_L2_CSLC-S1_V1": ["C2777443834-ASF", "C1259976861-ASF"], + "OPERA_L2_RTC-S1_V1": ["C2777436413-ASF", "C1259974840-ASF"], + "OPERA_L2_CSLC-S1-STATIC_PROVISIONAL_V0": ["C1258354200-ASF"], + "OPERA_L2_CSLC-S1-STATIC_V1": ["C1259982010-ASF"], + "OPERA_L2_CSLC-S1_PROVISIONAL_V0": ["C1257995185-ASF"], + "OPERA_L2_RTC-S1-STATIC_PROVISIONAL_V0": ["C1258354201-ASF"], + "OPERA_L2_RTC-S1-STATIC_V1": ["C1259981910-ASF"], + "OPERA_L2_RTC-S1_PROVISIONAL_V0": ["C1257995186-ASF"], + "OPERA_L2_CSLC-S1-STATIC_CALVAL_V1": ["C1260726384-ASF"], + "OPERA_L2_CSLC-S1_CALVAL_V1": ["C1260721945-ASF"], + "OPERA_L2_RTC-S1_CALVAL_V1": ["C1260721853-ASF"], + "OPERA_L2_RTC-S1-STATIC_CALVAL_V1": ["C1260726378-ASF"], + }, + "SLC-BURST": {"SENTINEL-1_BURSTS": ["C2709161906-ASF", "C1257024016-ASF"]}, + "ALOS PALSAR": { + "ALOS_PSR_RTC_HIGH": ["C1206487504-ASF", "C1207181535-ASF"], + "ALOS_PSR_L1.5": ["C1206485940-ASF", "C1205261223-ASF"], + "ALOS_PSR_RTC_LOW": ["C1206487217-ASF", "C1208013295-ASF"], + "ALOS_PSR_KMZ": ["C1206156901-ASF", "C1207019609-ASF"], + "ALOS_PSR_L1.0": ["C1206485320-ASF"], + "ALOS_PSR_L1.1": ["C1206485527-ASF", "C1207710476-ASF", "C1239611505-ASFDEV"], + "ALOS_PSR_L2.2": ["C2011599335-ASF", "C1239927797-ASF", "C1238733834-ASFDEV"], + "ALOS_PALSAR_INSAR_METADATA": ["C1229740239-ASF"], + }, + "ALOS AVNIR-2": { + "ALOS_AVNIR_OBS_ORI": [ + "C1808440897-ASF", + "C1233629671-ASF", + "C1234413224-ASFDEV", + ], + "ALOS_AVNIR_OBS_ORI_BROWSE": ["C1234712303-ASF"], + }, + "SIR-C": { + "STS-59_BROWSE_GRD": [ + "C1661710578-ASF", + "C1226557819-ASF", + "C1234413264-ASFDEV", + ], + "STS-59_BROWSE_SLC": [ + "C1661710581-ASF", + "C1226557809-ASF", + "C1234413265-ASFDEV", + ], + "STS-59_GRD": ["C1661710583-ASF", "C1226557808-ASF", "C1234413266-ASFDEV"], + "STS-59_META_GRD": ["C1661710586-ASF", "C1226557810-ASF", "C1234413267-ASFDEV"], + "STS-59_META_SLC": ["C1661710588-ASF", "C1226557811-ASF", "C1234413268-ASFDEV"], + "STS-59_SLC": ["C1661710590-ASF", "C1226557812-ASF", "C1234413269-ASFDEV"], + "STS-68_BROWSE_GRD": [ + "C1661710593-ASF", + "C1226557813-ASF", + "C1234413270-ASFDEV", + ], + "STS-68_BROWSE_SLC": [ + "C1661710596-ASF", + "C1226557814-ASF", + "C1234413271-ASFDEV", + ], + "STS-68_GRD": ["C1661710597-ASF", "C1226557815-ASF", "C1234413272-ASFDEV"], + "STS-68_META_GRD": ["C1661710600-ASF", "C1226557816-ASF", "C1234413273-ASFDEV"], + "STS-68_META_SLC": ["C1661710603-ASF", "C1226557817-ASF", "C1234413274-ASFDEV"], + "STS-68_SLC": ["C1661710604-ASF", "C1226557818-ASF", "C1234413275-ASFDEV"], + }, + "ARIA S1 GUNW": { + "SENTINEL-1_INTERFEROGRAMS": ["C1595422627-ASF", "C1225776654-ASF"], + "SENTINEL-1_INTERFEROGRAMS_AMPLITUDE": ["C1596065640-ASF", "C1225776655-ASF"], + "SENTINEL-1_INTERFEROGRAMS_COHERENCE": ["C1596065639-ASF", "C1225776657-ASF"], + "SENTINEL-1_INTERFEROGRAMS_CONNECTED_COMPONENTS": [ + "C1596065641-ASF", + "C1225776658-ASF", + ], + "SENTINEL-1_INTERFEROGRAMS_UNWRAPPED_PHASE": [ + "C1595765183-ASF", + "C1225776659-ASF", + ], + }, + "SMAP": { + "SPL1A_RO_METADATA_003": ["C1243122884-ASF", "C1233103964-ASF"], + "SPL1A_RO_QA_003": ["C1243124139-ASF", "C1216074923-ASF"], + "SPL1A_001": ["C1214473171-ASF", "C1212243761-ASF"], + "SPL1A_002": ["C1243149604-ASF", "C1213091807-ASF"], + "SPL1A_METADATA_001": ["C1214473426-ASF", "C1212243437-ASF"], + "SPL1A_METADATA_002": ["C1243119801-ASF", "C1213096699-ASF"], + "SPL1A_QA_001": ["C1214473839-ASF", "C1212249653-ASF"], + "SPL1A_QA_002": ["C1243133204-ASF", "C1213101573-ASF"], + "SPL1A_RO_001": ["C1243197402-ASF"], + "SPL1A_RO_002": ["C1243215430-ASF", "C1213136240-ASF"], + "SPL1A_RO_003": ["C1243124754-ASF", "C1216074755-ASF"], + "SPL1A_RO_METADATA_001": ["C1243141638-ASF", "C1213136752-ASF"], + "SPL1A_RO_METADATA_002": ["C1243162394-ASF", "C1213136799-ASF"], + "SPL1A_RO_QA_001": ["C1243168733-ASF", "C1213136709-ASF"], + "SPL1A_RO_QA_002": ["C1243168866-ASF", "C1213136844-ASF"], + "SPL1B_SO_LoRes_001": ["C1214473308-ASF", "C1212249811-ASF"], + "SPL1B_SO_LoRes_002": ["C1243253631-ASF", "C1213125007-ASF"], + "SPL1B_SO_LoRes_003": ["C1243133445-ASF", "C1216074919-ASF"], + "SPL1B_SO_LoRes_METADATA_001": ["C1214473550-ASF", "C1212196951-ASF"], + "SPL1B_SO_LoRes_METADATA_002": ["C1243197502-ASF", "C1213115690-ASF"], + "SPL1B_SO_LoRes_METADATA_003": ["C1243126328-ASF", "C1216074758-ASF"], + "SPL1B_SO_LoRes_QA_001": ["C1214474243-ASF", "C1212243666-ASF"], + "SPL1B_SO_LoRes_QA_002": ["C1243216659-ASF", "C1213115896-ASF"], + "SPL1B_SO_LoRes_QA_003": ["C1243129847-ASF", "C1216074761-ASF"], + "SPL1C_S0_HiRes_001": ["C1214473367-ASF", "C1212250364-ASF"], + "SPL1C_S0_HiRes_002": ["C1243268956-ASF", "C1213134622-ASF"], + "SPL1C_S0_HiRes_003": ["C1243144528-ASF", "C1216074770-ASF"], + "SPL1C_S0_HiRes_METADATA_001": ["C1214473624-ASF", "C1212246173-ASF"], + "SPL1C_S0_HiRes_METADATA_002": ["C1243228612-ASF", "C1213125156-ASF"], + "SPL1C_S0_HiRes_METADATA_003": ["C1243136142-ASF", "C1216074764-ASF"], + "SPL1C_S0_HiRes_QA_001": ["C1214474435-ASF", "C1212249773-ASF"], + "SPL1C_S0_HiRes_QA_002": ["C1243255360-ASF", "C1213134486-ASF"], + "SPL1C_S0_HiRes_QA_003": ["C1243140611-ASF", "C1233101609-ASF"], + "SPL1A_003": ["C1216074922-ASF"], + "SPL1A_METADATA_003": ["C1216074750-ASF"], + "SPL1A_QA_003": ["C1216074751-ASF"], + }, + "UAVSAR": { + "UAVSAR_POL_META": ["C1214353986-ASF", "C1210487703-ASF"], + "UAVSAR_INSAR_META": ["C1214336717-ASF", "C1212030772-ASF"], + "UAVSAR_INSAR_INT": ["C1214336045-ASF", "C1212001698-ASF"], + "UAVSAR_INSAR_AMP": ["C1214335430-ASF", "C1206116665-ASF"], + "UAVSAR_INSAR_AMP_GRD": ["C1214335471-ASF", "C1206132445-ASF"], + "UAVSAR_INSAR_DEM": ["C1214335903-ASF", "C1211962154-ASF"], + "UAVSAR_INSAR_INT_GRD": ["C1214336154-ASF", "C1212005594-ASF"], + "UAVSAR_INSAR_KMZ": ["C1214336554-ASF", "C1212019993-ASF"], + "UAVSAR_POL_DEM": ["C1214353593-ASF", "C1207638502-ASF"], + "UAVSAR_POL_INC": ["C1214353754-ASF", "C1210025872-ASF"], + "UAVSAR_POL_KMZ": ["C1214353859-ASF", "C1210485039-ASF"], + "UAVSAR_POL_ML_CMPLX_GRD": ["C1214337770-ASF", "C1207188317-ASF"], + "UAVSAR_POL_ML_CMPLX_GRD_3X3": ["C1214354144-ASF", "C1210546638-ASF"], + "UAVSAR_POL_ML_CMPLX_GRD_5X5": ["C1214354235-ASF", "C1206122195-ASF"], + "UAVSAR_POL_ML_CMPLX_SLANT": ["C1214343609-ASF", "C1209970710-ASF"], + "UAVSAR_POL_PAULI": ["C1214354031-ASF", "C1207038647-ASF"], + "UAVSAR_POL_SLOPE": ["C1214408428-ASF", "C1210599503-ASF"], + "UAVSAR_POL_STOKES": ["C1214419355-ASF", "C1210599673-ASF"], + }, + "RADARSAT-1": { + "RSAT-1_L0": ["C1206897141-ASF"], + "RSAT-1_L1": ["C1206936391-ASF", "C1205181982-ASF"], + "RSAT-1_POLAR_YEAR_ANTARCTICA_L1": ["C1215670813-ASF"], + "RSAT-1_POLAR_YEAR_GREENLAND_L0": ["C1215709884-ASF"], + "RSAT-1_POLAR_YEAR_GREENLAND_L1": ["C1215709880-ASF"], + "RSAT-1_POLAR_YEAR_KAMCHATKA_L1": ["C1215714443-ASF"], + "RSAT-1_POLAR_YEAR_SEA_ICE_MIN_MAX_L1": ["C1215775284-ASF"], + "RSAT-1_POLAR_YEAR_TOOLIK_L1": ["C1215614037-ASF"], + }, + "ERS": { + "ERS-1_L0": ["C1210197768-ASF", "C1205261222-ASF"], + "ERS-1_L1": ["C1211627521-ASF", "C1205302527-ASF"], + "ERS-2_L0": ["C1208794942-ASF", "C1207143701-ASF"], + "ERS-2_L1": ["C1209373626-ASF", "C1207144966-ASF"], + }, + "JERS-1": { + "JERS-1_L0": ["C1208662092-ASF", "C1207175327-ASF"], + "JERS-1_L1": ["C1207933168-ASF", "C1207177736-ASF"], + }, + "AIRSAR": { + "AIRSAR_POL_3FP": ["C1213921661-ASF", "C1205256880-ASF"], + "AIRSAR_INT_JPG": ["C1213921626-ASF", "C1000000306-ASF"], + "AIRSAR_POL_SYN_3FP": ["C1213928843-ASF", "C1208713702-ASF"], + "AIRSAR_TOP_C-DEM_STOKES": ["C1213927035-ASF", "C1208707768-ASF"], + "AIRSAR_TOP_DEM": ["C179001730-ASF", "C1208655639-ASF"], + "AIRSAR_TOP_DEM_C": ["C1213925022-ASF", "C1208680681-ASF"], + "AIRSAR_TOP_DEM_L": ["C1213926419-ASF", "C1208691361-ASF"], + "AIRSAR_TOP_DEM_P": ["C1213926777-ASF", "C1208703384-ASF"], + "AIRSAR_TOP_L-STOKES": ["C1213927939-ASF"], + "AIRSAR_TOP_P-STOKES": ["C1213928209-ASF"], + "AIRSAR_INT": ["C1208652494-ASF"], + }, + "SEASAT": { + "SEASAT_SAR_L1_TIFF": ["C1206500826-ASF", "C1206752770-ASF"], + "SEASAT_SAR_L1_HDF5": ["C1206500991-ASF", "C1206144699-ASF"], + }, } diff --git a/asf_search/CMR/translate.py b/asf_search/CMR/translate.py index 703bbfda..348a33dc 100644 --- a/asf_search/CMR/translate.py +++ b/asf_search/CMR/translate.py @@ -7,7 +7,7 @@ from shapely.geometry import Polygon from shapely.geometry.base import BaseGeometry from .field_map import field_map -from .datasets import platform_datasets +from .datasets import dataset_collections import logging @@ -53,8 +53,9 @@ def translate_opts(opts: ASFSearchOptions) -> list: dict_opts['collections'] = [] for dataset in dict_opts['datasets']: - if dataset_collections := platform_datasets.get(dataset): - dict_opts['collections'].extend(dataset_collections) + if collections_by_short_name := dataset_collections.get(dataset): + for shortName, concept_ids in collections_by_short_name.items(): + dict_opts['collections'].extend(concept_ids) else: raise ValueError(f'Could not find dataset named "{dataset}" provided for datasets keyword.') diff --git a/tests/Search/test_search.py b/tests/Search/test_search.py index 5b9f3989..d22a9f54 100644 --- a/tests/Search/test_search.py +++ b/tests/Search/test_search.py @@ -6,13 +6,9 @@ from asf_search.exceptions import ASFSearchError from asf_search.search import search from asf_search.ASFSearchResults import ASFSearchResults -from asf_search.CMR import platform_datasets - -import requests -from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_exception_type - +from asf_search.CMR import dataset_collections from pytest import raises - +import requests import requests_mock def run_test_ASFSearchResults(search_resp): @@ -78,39 +74,24 @@ def custom_matcher(request: requests.Request): with raises(ASFSearchError): results.raise_if_incomplete() -@retry( - reraise=True, - wait=wait_fixed(3), - stop=stop_after_attempt(3), - retry=retry_if_exception_type((requests.ReadTimeout, requests.HTTPError)) -) def test_datasets_search(): datasets = ['SENTINEL-1', 'RADARSAT-1', 'UAVSAR'] - should_raise_error = len([dataset for dataset in datasets if not platform_datasets.get(dataset)]) > 0 + should_raise_error = len([dataset for dataset in datasets if not dataset_collections.get(dataset)]) > 0 if should_raise_error: with raises(ValueError): search(datasets=datasets, maxResults=1) - - # get collection concept-ids from shortName cmr query else: - valid_collections = [] + valid_shortnames = [] for dataset in datasets: - valid_collections.extend(platform_datasets.get(dataset)) + valid_shortnames.extend([shortName for shortName in dataset_collections.get(dataset).keys()]) response = search(datasets=datasets, maxResults=250) - # Granules don't keep track of their collection concept-ids, - # but they do have the collection names we can use to find them! - shortNames = '&'.join(list(set([f"shortName[]={get(product.umm, 'CollectionReference', 'ShortName')}" for product in response]))) - - r = requests.get(f"https://cmr.earthdata.nasa.gov/search/collections.umm_json?{shortNames}&provider=ASF") - r.raise_for_status() - - items = r.json()['items'] - concept_ids = [item['meta']['concept-id'] for item in items] - - # check that results are limited to the expected datasets - for concept_id in concept_ids: - assert concept_id in valid_collections + # Get collection shortName of all granules + shortNames = list(set([get(product.umm, 'CollectionReference', 'ShortName') for product in response])) + + # and check that results are limited to the expected datasets by their shortname + for shortName in shortNames: + assert shortName in valid_shortnames \ No newline at end of file From c8ae701366eb32707b83dba14cda1f068a0ab8db Mon Sep 17 00:00:00 2001 From: kim Date: Tue, 7 Nov 2023 15:22:12 -0900 Subject: [PATCH 11/16] rolls dataset tests into project testing framework --- tests/Search/test_search.py | 30 +++++++++++++----------------- tests/pytest-config.yml | 5 +++++ tests/pytest-managers.py | 6 +++++- tests/yml_tests/test_search.yml | 12 ++++++++++++ 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/tests/Search/test_search.py b/tests/Search/test_search.py index d22a9f54..a6adb468 100644 --- a/tests/Search/test_search.py +++ b/tests/Search/test_search.py @@ -8,6 +8,7 @@ from asf_search.ASFSearchResults import ASFSearchResults from asf_search.CMR import dataset_collections from pytest import raises +from typing import List import requests import requests_mock @@ -74,24 +75,19 @@ def custom_matcher(request: requests.Request): with raises(ASFSearchError): results.raise_if_incomplete() -def test_datasets_search(): - datasets = ['SENTINEL-1', 'RADARSAT-1', 'UAVSAR'] - should_raise_error = len([dataset for dataset in datasets if not dataset_collections.get(dataset)]) > 0 - - if should_raise_error: +def run_test_datasets_search(datasets: List): + if any(dataset for dataset in datasets if dataset_collections.get(dataset) is None): with raises(ValueError): search(datasets=datasets, maxResults=1) - else: - valid_shortnames = [] + else: for dataset in datasets: - valid_shortnames.extend([shortName for shortName in dataset_collections.get(dataset).keys()]) - - response = search(datasets=datasets, maxResults=250) + valid_shortnames = list(dataset_collections.get(dataset)) - # Get collection shortName of all granules - shortNames = list(set([get(product.umm, 'CollectionReference', 'ShortName') for product in response])) - - # and check that results are limited to the expected datasets by their shortname - for shortName in shortNames: - assert shortName in valid_shortnames - \ No newline at end of file + response = search(datasets=dataset, maxResults=250) + + # Get collection shortName of all granules + shortNames = list(set([shortName for product in response if (shortName:=get(product.umm, 'CollectionReference', 'ShortName')) is not None])) + + # and check that results are limited to the expected datasets by their shortname + for shortName in shortNames: + assert shortName in valid_shortnames diff --git a/tests/pytest-config.yml b/tests/pytest-config.yml index b5dfbe6f..104c5226 100644 --- a/tests/pytest-config.yml +++ b/tests/pytest-config.yml @@ -199,6 +199,11 @@ test_types: required_keys: results method: test_output_format +- For running datasets keyword tests: + required_in_title: search-datasets + required_keys: datasets + method: test_search_datasets + - For running jupyter notebook example tests: required_keys: notebook method: test_notebook_examples diff --git a/tests/pytest-managers.py b/tests/pytest-managers.py index 70c58d83..0d3db18f 100644 --- a/tests/pytest-managers.py +++ b/tests/pytest-managers.py @@ -7,7 +7,7 @@ from ASFSearchResults.test_ASFSearchResults import run_test_output_format, run_test_ASFSearchResults_intersection from ASFSession.test_ASFSession import run_auth_with_cookiejar, run_auth_with_creds, run_auth_with_token, run_test_asf_session_rebuild_auth from BaselineSearch.test_baseline_search import * -from Search.test_search import run_test_ASFSearchResults, run_test_search, run_test_search_http_error +from Search.test_search import run_test_ASFSearchResults, run_test_datasets_search, run_test_search, run_test_search_http_error from Search.test_search_generator import run_test_search_generator, run_test_search_generator_multi from CMR.test_MissionList import run_test_get_project_names @@ -417,6 +417,10 @@ def test_ASFSearchResults_intersection(**kwargs) -> None: wkt = get_resource(kwargs['test_info']['wkt']) run_test_ASFSearchResults_intersection(wkt) +def test_search_datasets(**kwargs) -> None: + datasets = get_resource(kwargs['test_info']['datasets']) + run_test_datasets_search(datasets) + def test_serialization(**args) -> None: test_info = args['test_info'] product = get_resource(test_info.get('product')) diff --git a/tests/yml_tests/test_search.yml b/tests/yml_tests/test_search.yml index 97449d63..4da5f9dc 100644 --- a/tests/yml_tests/test_search.yml +++ b/tests/yml_tests/test_search.yml @@ -42,3 +42,15 @@ tests: platform: "Sentinel-1" status_code: 500 report: "Server Error: This is a Test Error" + +- test-search-datasets S1 Datasets: + datasets: ['SENTINEL-1', 'SLC-BURST', 'OPERA-S1'] + +- test-search-datasets S1 Datasets and non-S1: + datasets: ['SENTINEL-1', 'SLC-BURST', 'OPERA-S1', 'UAVSAR'] + +- test-search-datasets fake dataset: + datasets: 'FAKE-DATASET-V1' + +- test-search-datasets S1 Datasets and fake dataset: + datasets: ['SENTINEL-1', 'SLC-BURST', 'OPERA-S1', 'FAKE-DATASET-V2'] From 076ecdcba6967b17d7f97ce26e2600be33237c6b Mon Sep 17 00:00:00 2001 From: kim Date: Tue, 7 Nov 2023 16:56:17 -0900 Subject: [PATCH 12/16] adds check for S1/ALOS dataset to should_use_asf_frame() --- asf_search/CMR/translate.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/asf_search/CMR/translate.py b/asf_search/CMR/translate.py index 348a33dc..a71229ac 100644 --- a/asf_search/CMR/translate.py +++ b/asf_search/CMR/translate.py @@ -102,9 +102,16 @@ def translate_opts(opts: ASFSearchOptions) -> list: def should_use_asf_frame(cmr_opts): asf_frame_platforms = ['SENTINEL-1A', 'SENTINEL-1B', 'ALOS'] + asf_frame_datasets = ['SENTINEL-1', 'OPERA-S1', 'SLC-BURST', 'ALOS PALSAR', 'ALOS AVNIR-2'] + + asf_frame_collections = [] + for dataset in asf_frame_datasets: + for concept_ids in dataset_collections.get(dataset).values(): + asf_frame_collections.extend(concept_ids) return any([ p[0] == 'platform[]' and p[1].upper() in asf_frame_platforms + or p[0] == 'echo_collection_id[]' and p[1] in asf_frame_collections for p in cmr_opts ]) From 0eda5ad26ee9961e1cdeb1b1c3b13f050c8a4be1 Mon Sep 17 00:00:00 2001 From: kim Date: Wed, 8 Nov 2023 09:49:03 -0900 Subject: [PATCH 13/16] changes 'datasets' to 'dataset' --- CHANGELOG.md | 2 +- asf_search/ASFSearchOptions/validator_map.py | 2 +- asf_search/CMR/subquery.py | 2 +- asf_search/CMR/translate.py | 10 +++++----- asf_search/search/search.py | 2 +- asf_search/search/search_generator.py | 2 +- tests/pytest-config.yml | 8 ++++---- tests/pytest-managers.py | 8 ++++---- tests/yml_tests/test_search.yml | 16 ++++++++-------- 9 files changed, 26 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6156cf54..2ee8cfb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ------ ## [v6.6.4](https://github.com/asfadmin/Discovery-asf_search/compare/v6.6.3...v6.6.4) ### Added -- Adds new `datasets` keyword to `search()` as an alternative to `platform`. Allows users to get results from multiple platforms at once in a single page. +- Adds new `dataset` keyword to `search()` as an alternative to `platform`. Allows users to get results from multiple platforms at once in a single page. ### Changed - Changes `CMR_FORMAT_EXT` constant from `umm_json_v1_4` to `umm_json`, umm returned from CMR will now be in latest umm format by default diff --git a/asf_search/ASFSearchOptions/validator_map.py b/asf_search/ASFSearchOptions/validator_map.py index 65972b4c..45414bc8 100644 --- a/asf_search/ASFSearchOptions/validator_map.py +++ b/asf_search/ASFSearchOptions/validator_map.py @@ -57,7 +57,7 @@ def validate(key, value): 'instrument': parse_string, 'collections': parse_string_list, 'temporalBaselineDays': parse_string_list, - 'datasets': parse_string_list, + 'dataset': parse_string_list, 'absoluteBurstID': parse_int_list, 'relativeBurstID': parse_int_list, 'fullBurstID': parse_string_list, diff --git a/asf_search/CMR/subquery.py b/asf_search/CMR/subquery.py index 8c00c789..42bc49cd 100644 --- a/asf_search/CMR/subquery.py +++ b/asf_search/CMR/subquery.py @@ -22,7 +22,7 @@ def build_subqueries(opts: ASFSearchOptions) -> List[ASFSearchOptions]: if params.get('product_list') is not None: params['product_list'] = chunk_list(params['product_list'], CMR_PAGE_SIZE) - list_param_names = ['platform', 'season', 'collections', 'datasets'] # these parameters will dodge the subquery system + list_param_names = ['platform', 'season', 'collections', 'dataset'] # these parameters will dodge the subquery system skip_param_names = ['maxResults']# these params exist in opts, but shouldn't be passed on to subqueries at ALL params = dict([ (k, v) for k, v in params.items() if k not in skip_param_names ]) diff --git a/asf_search/CMR/translate.py b/asf_search/CMR/translate.py index a71229ac..e38fe193 100644 --- a/asf_search/CMR/translate.py +++ b/asf_search/CMR/translate.py @@ -48,18 +48,18 @@ def translate_opts(opts: ASFSearchOptions) -> list: if any(key in dict_opts for key in ['start', 'end', 'season']): dict_opts = fix_date(dict_opts) - if 'datasets' in dict_opts: + if 'dataset' in dict_opts: if 'collections' not in dict_opts: dict_opts['collections'] = [] - for dataset in dict_opts['datasets']: + for dataset in dict_opts['dataset']: if collections_by_short_name := dataset_collections.get(dataset): - for shortName, concept_ids in collections_by_short_name.items(): + for concept_ids in collections_by_short_name.values(): dict_opts['collections'].extend(concept_ids) else: - raise ValueError(f'Could not find dataset named "{dataset}" provided for datasets keyword.') + raise ValueError(f'Could not find dataset named "{dataset}" provided for dataset keyword.') - dict_opts.pop('datasets') + dict_opts.pop('dataset') # convert the above parameters to a list of key/value tuples cmr_opts = [] diff --git a/asf_search/search/search.py b/asf_search/search/search.py index 8603038a..49f229ee 100644 --- a/asf_search/search/search.py +++ b/asf_search/search/search.py @@ -40,7 +40,7 @@ def search( fullBurstID: Union[str, Iterable[str]] = None, collections: Union[str, Iterable[str]] = None, temporalBaselineDays: Union[str, Iterable[str]] = None, - datasets: Union[str, Iterable[str]] = None, + dataset: Union[str, Iterable[str]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> ASFSearchResults: diff --git a/asf_search/search/search_generator.py b/asf_search/search/search_generator.py index 4ddcde1e..e94d3916 100644 --- a/asf_search/search/search_generator.py +++ b/asf_search/search/search_generator.py @@ -54,7 +54,7 @@ def search_generator( fullBurstID: Union[str, Iterable[str]] = None, collections: Union[str, Iterable[str]] = None, temporalBaselineDays: Union[str, Iterable[str]] = None, - datasets: Union[str, Iterable[str]] = None, + dataset: Union[str, Iterable[str]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> Generator[ASFSearchResults, None, None]: diff --git a/tests/pytest-config.yml b/tests/pytest-config.yml index 104c5226..285b9fe3 100644 --- a/tests/pytest-config.yml +++ b/tests/pytest-config.yml @@ -199,10 +199,10 @@ test_types: required_keys: results method: test_output_format -- For running datasets keyword tests: - required_in_title: search-datasets - required_keys: datasets - method: test_search_datasets +- For running dataset keyword tests: + required_in_title: search-dataset + required_keys: dataset + method: test_search_dataset - For running jupyter notebook example tests: required_keys: notebook diff --git a/tests/pytest-managers.py b/tests/pytest-managers.py index 0d3db18f..438b1520 100644 --- a/tests/pytest-managers.py +++ b/tests/pytest-managers.py @@ -7,7 +7,7 @@ from ASFSearchResults.test_ASFSearchResults import run_test_output_format, run_test_ASFSearchResults_intersection from ASFSession.test_ASFSession import run_auth_with_cookiejar, run_auth_with_creds, run_auth_with_token, run_test_asf_session_rebuild_auth from BaselineSearch.test_baseline_search import * -from Search.test_search import run_test_ASFSearchResults, run_test_datasets_search, run_test_search, run_test_search_http_error +from Search.test_search import run_test_ASFSearchResults, run_test_dataset_search, run_test_search, run_test_search_http_error from Search.test_search_generator import run_test_search_generator, run_test_search_generator_multi from CMR.test_MissionList import run_test_get_project_names @@ -417,9 +417,9 @@ def test_ASFSearchResults_intersection(**kwargs) -> None: wkt = get_resource(kwargs['test_info']['wkt']) run_test_ASFSearchResults_intersection(wkt) -def test_search_datasets(**kwargs) -> None: - datasets = get_resource(kwargs['test_info']['datasets']) - run_test_datasets_search(datasets) +def test_search_dataset(**kwargs) -> None: + dataset = get_resource(kwargs['test_info']['dataset']) + run_test_dataset_search(dataset) def test_serialization(**args) -> None: test_info = args['test_info'] diff --git a/tests/yml_tests/test_search.yml b/tests/yml_tests/test_search.yml index 4da5f9dc..2e336bcd 100644 --- a/tests/yml_tests/test_search.yml +++ b/tests/yml_tests/test_search.yml @@ -43,14 +43,14 @@ tests: status_code: 500 report: "Server Error: This is a Test Error" -- test-search-datasets S1 Datasets: - datasets: ['SENTINEL-1', 'SLC-BURST', 'OPERA-S1'] +- test-search-dataset S1 Datasets: + dataset: ['SENTINEL-1', 'SLC-BURST', 'OPERA-S1'] -- test-search-datasets S1 Datasets and non-S1: - datasets: ['SENTINEL-1', 'SLC-BURST', 'OPERA-S1', 'UAVSAR'] +- test-search-dataset S1 Datasets and non-S1: + dataset: ['SENTINEL-1', 'SLC-BURST', 'OPERA-S1', 'UAVSAR'] -- test-search-datasets fake dataset: - datasets: 'FAKE-DATASET-V1' +- test-search-dataset fake dataset: + dataset: 'FAKE-DATASET-V1' -- test-search-datasets S1 Datasets and fake dataset: - datasets: ['SENTINEL-1', 'SLC-BURST', 'OPERA-S1', 'FAKE-DATASET-V2'] +- test-search-dataset S1 Datasets and fake dataset: + dataset: ['SENTINEL-1', 'SLC-BURST', 'OPERA-S1', 'FAKE-DATASET-V2'] From 2a9cc9dd1814a36b42fcd00849cf396ecf7b6567 Mon Sep 17 00:00:00 2001 From: kim Date: Wed, 8 Nov 2023 10:26:14 -0900 Subject: [PATCH 14/16] update test name --- tests/Search/test_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Search/test_search.py b/tests/Search/test_search.py index a6adb468..e11595fe 100644 --- a/tests/Search/test_search.py +++ b/tests/Search/test_search.py @@ -75,7 +75,7 @@ def custom_matcher(request: requests.Request): with raises(ASFSearchError): results.raise_if_incomplete() -def run_test_datasets_search(datasets: List): +def run_test_dataset_search(datasets: List): if any(dataset for dataset in datasets if dataset_collections.get(dataset) is None): with raises(ValueError): search(datasets=datasets, maxResults=1) From fddc5440348451975818d62044b2f01c56796de4 Mon Sep 17 00:00:00 2001 From: kim Date: Wed, 8 Nov 2023 11:21:30 -0900 Subject: [PATCH 15/16] fixes search test using "datasets" --- tests/Search/test_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Search/test_search.py b/tests/Search/test_search.py index e11595fe..449e6d4f 100644 --- a/tests/Search/test_search.py +++ b/tests/Search/test_search.py @@ -83,7 +83,7 @@ def run_test_dataset_search(datasets: List): for dataset in datasets: valid_shortnames = list(dataset_collections.get(dataset)) - response = search(datasets=dataset, maxResults=250) + response = search(dataset=dataset, maxResults=250) # Get collection shortName of all granules shortNames = list(set([shortName for product in response if (shortName:=get(product.umm, 'CollectionReference', 'ShortName')) is not None])) From 12a317203c05737da2827402fbf7f9903c9a3022 Mon Sep 17 00:00:00 2001 From: kim Date: Wed, 8 Nov 2023 11:35:49 -0900 Subject: [PATCH 16/16] fixes last instance of "datasets" --- tests/Search/test_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Search/test_search.py b/tests/Search/test_search.py index 449e6d4f..463d834a 100644 --- a/tests/Search/test_search.py +++ b/tests/Search/test_search.py @@ -78,7 +78,7 @@ def custom_matcher(request: requests.Request): def run_test_dataset_search(datasets: List): if any(dataset for dataset in datasets if dataset_collections.get(dataset) is None): with raises(ValueError): - search(datasets=datasets, maxResults=1) + search(dataset=datasets, maxResults=1) else: for dataset in datasets: valid_shortnames = list(dataset_collections.get(dataset))