diff --git a/CHANGELOG.md b/CHANGELOG.md index befca2d7..2ee8cfb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,12 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). --> ------ ## [v6.6.4](https://github.com/asfadmin/Discovery-asf_search/compare/v6.6.3...v6.6.4) +### Added +- Adds new `dataset` keyword to `search()` as an alternative to `platform`. Allows users to get results from multiple platforms at once in a single page. + +### Changed +- Changes `CMR_FORMAT_EXT` constant from `umm_json_v1_4` to `umm_json`, umm returned from CMR will now be in latest umm format by default + ### Fixed - ERS-1, ERS-2, JERS-1, and RADARSAT-1 now assign `FRAME_NUMBER` to the `frameNumber` properties field diff --git a/asf_search/ASFSearchOptions/validator_map.py b/asf_search/ASFSearchOptions/validator_map.py index 64162bc1..45414bc8 100644 --- a/asf_search/ASFSearchOptions/validator_map.py +++ b/asf_search/ASFSearchOptions/validator_map.py @@ -57,12 +57,13 @@ def validate(key, value): 'instrument': parse_string, 'collections': parse_string_list, 'temporalBaselineDays': parse_string_list, - + 'dataset': parse_string_list, + 'absoluteBurstID': parse_int_list, + 'relativeBurstID': parse_int_list, + 'fullBurstID': parse_string_list, + # Config parameters Parser 'session': parse_session, 'host': parse_string, - 'provider': parse_string, - 'absoluteBurstID': parse_int_list, - 'relativeBurstID': parse_int_list, - 'fullBurstID': parse_string_list + 'provider': parse_string } diff --git a/asf_search/CMR/__init__.py b/asf_search/CMR/__init__.py index f382ebda..905e8eb3 100644 --- a/asf_search/CMR/__init__.py +++ b/asf_search/CMR/__init__.py @@ -1,4 +1,5 @@ from .MissionList import get_campaigns from .subquery import build_subqueries from .translate import translate_product, translate_opts, get_additional_fields -from .field_map import field_map \ No newline at end of file +from .field_map import field_map +from .datasets import dataset_collections \ No newline at end of file diff --git a/asf_search/CMR/datasets.py b/asf_search/CMR/datasets.py new file mode 100644 index 00000000..b9bf61b6 --- /dev/null +++ b/asf_search/CMR/datasets.py @@ -0,0 +1,327 @@ +dataset_collections = { + "SENTINEL-1": { + "SENTINEL-1A_SLC": ["C1214470488-ASF", "C1205428742-ASF", "C1234413245-ASFDEV"], + "SENTINEL-1B_SLC": ["C1327985661-ASF", "C1216244348-ASF", "C1234413263-ASFDEV"], + "SENTINEL-1A_DP_GRD_HIGH": [ + "C1214470533-ASF", + "C1212201032-ASF", + "C1234413229-ASFDEV", + ], + "SENTINEL-1A_DP_META_GRD_HIGH": [ + "C1214470576-ASF", + "C1212209226-ASF", + "C1234413232-ASFDEV", + ], + "SENTINEL-1B_DP_GRD_HIGH": [ + "C1327985645-ASF", + "C1216244589-ASF", + "C1234413247-ASFDEV", + ], + "SENTINEL-1A_META_SLC": [ + "C1214470496-ASF", + "C1208117434-ASF", + "C1234413236-ASFDEV", + ], + "SENTINEL-1A_META_RAW": [ + "C1214470532-ASF", + "C1208115009-ASF", + "C1234413235-ASFDEV", + ], + "SENTINEL-1A_OCN": ["C1214472977-ASF", "C1212212560-ASF", "C1234413237-ASFDEV"], + "SENTINEL-1A_DP_META_GRD_MEDIUM": [ + "C1214472336-ASF", + "C1212212493-ASF", + "C1234413233-ASFDEV", + ], + "SENTINEL-1A_META_OCN": [ + "C1266376001-ASF", + "C1215704763-ASF", + "C1234413234-ASFDEV", + ], + "SENTINEL-1A_SP_META_GRD_HIGH": [ + "C1214470732-ASF", + "C1212158326-ASF", + "C1234413243-ASFDEV", + ], + "SENTINEL-1B_DP_GRD_MEDIUM": [ + "C1327985660-ASF", + "C1216244594-ASF", + "C1234413248-ASFDEV", + ], + "SENTINEL-1B_DP_META_GRD_HIGH": [ + "C1327985741-ASF", + "C1216244601-ASF", + "C1234413250-ASFDEV", + ], + "SENTINEL-1B_DP_META_GRD_MEDIUM": [ + "C1327985578-ASF", + "C1216244591-ASF", + "C1234413251-ASFDEV", + ], + "SENTINEL-1B_META_RAW": [ + "C1327985650-ASF", + "C1216244595-ASF", + "C1234413253-ASFDEV", + ], + "SENTINEL-1B_META_SLC": [ + "C1327985617-ASF", + "C1216244585-ASF", + "C1234413254-ASFDEV", + ], + "SENTINEL-1B_OCN": ["C1327985579-ASF", "C1216244593-ASF", "C1234413255-ASFDEV"], + "SENTINEL-1B_SP_META_GRD_HIGH": [ + "C1327985619-ASF", + "C1216244587-ASF", + "C1234413261-ASFDEV", + ], + "SENTINEL-1A_SP_GRD_MEDIUM": [ + "C1214472994-ASF", + "C1212158318-ASF", + "C1234413241-ASFDEV", + ], + "SENTINEL-1A_SP_META_GRD_MEDIUM": [ + "C1214473170-ASF", + "C1212233976-ASF", + "C1234413244-ASFDEV", + ], + "SENTINEL-1B_META_OCN": [ + "C1327985646-ASF", + "C1216244590-ASF", + "C1234413252-ASFDEV", + ], + "SENTINEL-1B_SP_GRD_MEDIUM": [ + "C1327985740-ASF", + "C1216244600-ASF", + "C1234413259-ASFDEV", + ], + "SENTINEL-1B_SP_META_GRD_MEDIUM": [ + "C1327985739-ASF", + "C1216244598-ASF", + "C1234413262-ASFDEV", + ], + "SENTINEL-1A_RAW": ["C1214470561-ASF", "C1205264459-ASF", "C1234413238-ASFDEV"], + "SENTINEL-1A_DP_GRD_MEDIUM": [ + "C1214471521-ASF", + "C1212209035-ASF", + "C1234413230-ASFDEV", + ], + "SENTINEL-1A_SP_GRD_HIGH": [ + "C1214470682-ASF", + "C1212158327-ASF", + "C1234413240-ASFDEV", + ], + "SENTINEL-1B_RAW": ["C1327985647-ASF", "C1216244592-ASF", "C1234413256-ASFDEV"], + "SENTINEL-1A_DP_GRD_FULL": [ + "C1214471197-ASF", + "C1212200781-ASF", + "C1234413228-ASFDEV", + ], + "SENTINEL-1A_DP_META_GRD_FULL": [ + "C1214471960-ASF", + "C1212209075-ASF", + "C1234413231-ASFDEV", + ], + "SENTINEL-1A_SP_GRD_FULL": ["C1214472978-ASF", "C1234413239-ASFDEV"], + "SENTINEL-1A_SP_META_GRD_FULL": ["C1214473165-ASF", "C1234413242-ASFDEV"], + "SENTINEL-1B_DP_GRD_FULL": [ + "C1327985697-ASF", + "C1216244597-ASF", + "C1234413246-ASFDEV", + ], + "SENTINEL-1B_DP_META_GRD_FULL": [ + "C1327985651-ASF", + "C1216244596-ASF", + "C1234413249-ASFDEV", + ], + "SENTINEL-1B_SP_GRD_FULL": [ + "C1327985644-ASF", + "C1216244588-ASF", + "C1234413257-ASFDEV", + ], + "SENTINEL-1B_SP_GRD_HIGH": [ + "C1327985571-ASF", + "C1216244586-ASF", + "C1234413258-ASFDEV", + ], + "SENTINEL-1B_SP_META_GRD_FULL": [ + "C1327985674-ASF", + "C1216244599-ASF", + "C1234413260-ASFDEV", + ], + "S1_Bursts": ["C1244552887-ASFDEV"], + "SENTINEL-1_BURSTS_DEV10": ["C1257175154-ASFDEV"], + "Sentinel-1_Burst_Map": ["C1244598379-ASFDEV"], + "Various Browse Images": ["C1240784657-ASFDEV"], + }, + "OPERA-S1": { + "OPERA_L2_CSLC-S1_V1": ["C2777443834-ASF", "C1259976861-ASF"], + "OPERA_L2_RTC-S1_V1": ["C2777436413-ASF", "C1259974840-ASF"], + "OPERA_L2_CSLC-S1-STATIC_PROVISIONAL_V0": ["C1258354200-ASF"], + "OPERA_L2_CSLC-S1-STATIC_V1": ["C1259982010-ASF"], + "OPERA_L2_CSLC-S1_PROVISIONAL_V0": ["C1257995185-ASF"], + "OPERA_L2_RTC-S1-STATIC_PROVISIONAL_V0": ["C1258354201-ASF"], + "OPERA_L2_RTC-S1-STATIC_V1": ["C1259981910-ASF"], + "OPERA_L2_RTC-S1_PROVISIONAL_V0": ["C1257995186-ASF"], + "OPERA_L2_CSLC-S1-STATIC_CALVAL_V1": ["C1260726384-ASF"], + "OPERA_L2_CSLC-S1_CALVAL_V1": ["C1260721945-ASF"], + "OPERA_L2_RTC-S1_CALVAL_V1": ["C1260721853-ASF"], + "OPERA_L2_RTC-S1-STATIC_CALVAL_V1": ["C1260726378-ASF"], + }, + "SLC-BURST": {"SENTINEL-1_BURSTS": ["C2709161906-ASF", "C1257024016-ASF"]}, + "ALOS PALSAR": { + "ALOS_PSR_RTC_HIGH": ["C1206487504-ASF", "C1207181535-ASF"], + "ALOS_PSR_L1.5": ["C1206485940-ASF", "C1205261223-ASF"], + "ALOS_PSR_RTC_LOW": ["C1206487217-ASF", "C1208013295-ASF"], + "ALOS_PSR_KMZ": ["C1206156901-ASF", "C1207019609-ASF"], + "ALOS_PSR_L1.0": ["C1206485320-ASF"], + "ALOS_PSR_L1.1": ["C1206485527-ASF", "C1207710476-ASF", "C1239611505-ASFDEV"], + "ALOS_PSR_L2.2": ["C2011599335-ASF", "C1239927797-ASF", "C1238733834-ASFDEV"], + "ALOS_PALSAR_INSAR_METADATA": ["C1229740239-ASF"], + }, + "ALOS AVNIR-2": { + "ALOS_AVNIR_OBS_ORI": [ + "C1808440897-ASF", + "C1233629671-ASF", + "C1234413224-ASFDEV", + ], + "ALOS_AVNIR_OBS_ORI_BROWSE": ["C1234712303-ASF"], + }, + "SIR-C": { + "STS-59_BROWSE_GRD": [ + "C1661710578-ASF", + "C1226557819-ASF", + "C1234413264-ASFDEV", + ], + "STS-59_BROWSE_SLC": [ + "C1661710581-ASF", + "C1226557809-ASF", + "C1234413265-ASFDEV", + ], + "STS-59_GRD": ["C1661710583-ASF", "C1226557808-ASF", "C1234413266-ASFDEV"], + "STS-59_META_GRD": ["C1661710586-ASF", "C1226557810-ASF", "C1234413267-ASFDEV"], + "STS-59_META_SLC": ["C1661710588-ASF", "C1226557811-ASF", "C1234413268-ASFDEV"], + "STS-59_SLC": ["C1661710590-ASF", "C1226557812-ASF", "C1234413269-ASFDEV"], + "STS-68_BROWSE_GRD": [ + "C1661710593-ASF", + "C1226557813-ASF", + "C1234413270-ASFDEV", + ], + "STS-68_BROWSE_SLC": [ + "C1661710596-ASF", + "C1226557814-ASF", + "C1234413271-ASFDEV", + ], + "STS-68_GRD": ["C1661710597-ASF", "C1226557815-ASF", "C1234413272-ASFDEV"], + "STS-68_META_GRD": ["C1661710600-ASF", "C1226557816-ASF", "C1234413273-ASFDEV"], + "STS-68_META_SLC": ["C1661710603-ASF", "C1226557817-ASF", "C1234413274-ASFDEV"], + "STS-68_SLC": ["C1661710604-ASF", "C1226557818-ASF", "C1234413275-ASFDEV"], + }, + "ARIA S1 GUNW": { + "SENTINEL-1_INTERFEROGRAMS": ["C1595422627-ASF", "C1225776654-ASF"], + "SENTINEL-1_INTERFEROGRAMS_AMPLITUDE": ["C1596065640-ASF", "C1225776655-ASF"], + "SENTINEL-1_INTERFEROGRAMS_COHERENCE": ["C1596065639-ASF", "C1225776657-ASF"], + "SENTINEL-1_INTERFEROGRAMS_CONNECTED_COMPONENTS": [ + "C1596065641-ASF", + "C1225776658-ASF", + ], + "SENTINEL-1_INTERFEROGRAMS_UNWRAPPED_PHASE": [ + "C1595765183-ASF", + "C1225776659-ASF", + ], + }, + "SMAP": { + "SPL1A_RO_METADATA_003": ["C1243122884-ASF", "C1233103964-ASF"], + "SPL1A_RO_QA_003": ["C1243124139-ASF", "C1216074923-ASF"], + "SPL1A_001": ["C1214473171-ASF", "C1212243761-ASF"], + "SPL1A_002": ["C1243149604-ASF", "C1213091807-ASF"], + "SPL1A_METADATA_001": ["C1214473426-ASF", "C1212243437-ASF"], + "SPL1A_METADATA_002": ["C1243119801-ASF", "C1213096699-ASF"], + "SPL1A_QA_001": ["C1214473839-ASF", "C1212249653-ASF"], + "SPL1A_QA_002": ["C1243133204-ASF", "C1213101573-ASF"], + "SPL1A_RO_001": ["C1243197402-ASF"], + "SPL1A_RO_002": ["C1243215430-ASF", "C1213136240-ASF"], + "SPL1A_RO_003": ["C1243124754-ASF", "C1216074755-ASF"], + "SPL1A_RO_METADATA_001": ["C1243141638-ASF", "C1213136752-ASF"], + "SPL1A_RO_METADATA_002": ["C1243162394-ASF", "C1213136799-ASF"], + "SPL1A_RO_QA_001": ["C1243168733-ASF", "C1213136709-ASF"], + "SPL1A_RO_QA_002": ["C1243168866-ASF", "C1213136844-ASF"], + "SPL1B_SO_LoRes_001": ["C1214473308-ASF", "C1212249811-ASF"], + "SPL1B_SO_LoRes_002": ["C1243253631-ASF", "C1213125007-ASF"], + "SPL1B_SO_LoRes_003": ["C1243133445-ASF", "C1216074919-ASF"], + "SPL1B_SO_LoRes_METADATA_001": ["C1214473550-ASF", "C1212196951-ASF"], + "SPL1B_SO_LoRes_METADATA_002": ["C1243197502-ASF", "C1213115690-ASF"], + "SPL1B_SO_LoRes_METADATA_003": ["C1243126328-ASF", "C1216074758-ASF"], + "SPL1B_SO_LoRes_QA_001": ["C1214474243-ASF", "C1212243666-ASF"], + "SPL1B_SO_LoRes_QA_002": ["C1243216659-ASF", "C1213115896-ASF"], + "SPL1B_SO_LoRes_QA_003": ["C1243129847-ASF", "C1216074761-ASF"], + "SPL1C_S0_HiRes_001": ["C1214473367-ASF", "C1212250364-ASF"], + "SPL1C_S0_HiRes_002": ["C1243268956-ASF", "C1213134622-ASF"], + "SPL1C_S0_HiRes_003": ["C1243144528-ASF", "C1216074770-ASF"], + "SPL1C_S0_HiRes_METADATA_001": ["C1214473624-ASF", "C1212246173-ASF"], + "SPL1C_S0_HiRes_METADATA_002": ["C1243228612-ASF", "C1213125156-ASF"], + "SPL1C_S0_HiRes_METADATA_003": ["C1243136142-ASF", "C1216074764-ASF"], + "SPL1C_S0_HiRes_QA_001": ["C1214474435-ASF", "C1212249773-ASF"], + "SPL1C_S0_HiRes_QA_002": ["C1243255360-ASF", "C1213134486-ASF"], + "SPL1C_S0_HiRes_QA_003": ["C1243140611-ASF", "C1233101609-ASF"], + "SPL1A_003": ["C1216074922-ASF"], + "SPL1A_METADATA_003": ["C1216074750-ASF"], + "SPL1A_QA_003": ["C1216074751-ASF"], + }, + "UAVSAR": { + "UAVSAR_POL_META": ["C1214353986-ASF", "C1210487703-ASF"], + "UAVSAR_INSAR_META": ["C1214336717-ASF", "C1212030772-ASF"], + "UAVSAR_INSAR_INT": ["C1214336045-ASF", "C1212001698-ASF"], + "UAVSAR_INSAR_AMP": ["C1214335430-ASF", "C1206116665-ASF"], + "UAVSAR_INSAR_AMP_GRD": ["C1214335471-ASF", "C1206132445-ASF"], + "UAVSAR_INSAR_DEM": ["C1214335903-ASF", "C1211962154-ASF"], + "UAVSAR_INSAR_INT_GRD": ["C1214336154-ASF", "C1212005594-ASF"], + "UAVSAR_INSAR_KMZ": ["C1214336554-ASF", "C1212019993-ASF"], + "UAVSAR_POL_DEM": ["C1214353593-ASF", "C1207638502-ASF"], + "UAVSAR_POL_INC": ["C1214353754-ASF", "C1210025872-ASF"], + "UAVSAR_POL_KMZ": ["C1214353859-ASF", "C1210485039-ASF"], + "UAVSAR_POL_ML_CMPLX_GRD": ["C1214337770-ASF", "C1207188317-ASF"], + "UAVSAR_POL_ML_CMPLX_GRD_3X3": ["C1214354144-ASF", "C1210546638-ASF"], + "UAVSAR_POL_ML_CMPLX_GRD_5X5": ["C1214354235-ASF", "C1206122195-ASF"], + "UAVSAR_POL_ML_CMPLX_SLANT": ["C1214343609-ASF", "C1209970710-ASF"], + "UAVSAR_POL_PAULI": ["C1214354031-ASF", "C1207038647-ASF"], + "UAVSAR_POL_SLOPE": ["C1214408428-ASF", "C1210599503-ASF"], + "UAVSAR_POL_STOKES": ["C1214419355-ASF", "C1210599673-ASF"], + }, + "RADARSAT-1": { + "RSAT-1_L0": ["C1206897141-ASF"], + "RSAT-1_L1": ["C1206936391-ASF", "C1205181982-ASF"], + "RSAT-1_POLAR_YEAR_ANTARCTICA_L1": ["C1215670813-ASF"], + "RSAT-1_POLAR_YEAR_GREENLAND_L0": ["C1215709884-ASF"], + "RSAT-1_POLAR_YEAR_GREENLAND_L1": ["C1215709880-ASF"], + "RSAT-1_POLAR_YEAR_KAMCHATKA_L1": ["C1215714443-ASF"], + "RSAT-1_POLAR_YEAR_SEA_ICE_MIN_MAX_L1": ["C1215775284-ASF"], + "RSAT-1_POLAR_YEAR_TOOLIK_L1": ["C1215614037-ASF"], + }, + "ERS": { + "ERS-1_L0": ["C1210197768-ASF", "C1205261222-ASF"], + "ERS-1_L1": ["C1211627521-ASF", "C1205302527-ASF"], + "ERS-2_L0": ["C1208794942-ASF", "C1207143701-ASF"], + "ERS-2_L1": ["C1209373626-ASF", "C1207144966-ASF"], + }, + "JERS-1": { + "JERS-1_L0": ["C1208662092-ASF", "C1207175327-ASF"], + "JERS-1_L1": ["C1207933168-ASF", "C1207177736-ASF"], + }, + "AIRSAR": { + "AIRSAR_POL_3FP": ["C1213921661-ASF", "C1205256880-ASF"], + "AIRSAR_INT_JPG": ["C1213921626-ASF", "C1000000306-ASF"], + "AIRSAR_POL_SYN_3FP": ["C1213928843-ASF", "C1208713702-ASF"], + "AIRSAR_TOP_C-DEM_STOKES": ["C1213927035-ASF", "C1208707768-ASF"], + "AIRSAR_TOP_DEM": ["C179001730-ASF", "C1208655639-ASF"], + "AIRSAR_TOP_DEM_C": ["C1213925022-ASF", "C1208680681-ASF"], + "AIRSAR_TOP_DEM_L": ["C1213926419-ASF", "C1208691361-ASF"], + "AIRSAR_TOP_DEM_P": ["C1213926777-ASF", "C1208703384-ASF"], + "AIRSAR_TOP_L-STOKES": ["C1213927939-ASF"], + "AIRSAR_TOP_P-STOKES": ["C1213928209-ASF"], + "AIRSAR_INT": ["C1208652494-ASF"], + }, + "SEASAT": { + "SEASAT_SAR_L1_TIFF": ["C1206500826-ASF", "C1206752770-ASF"], + "SEASAT_SAR_L1_HDF5": ["C1206500991-ASF", "C1206144699-ASF"], + }, +} diff --git a/asf_search/CMR/subquery.py b/asf_search/CMR/subquery.py index f37a0ef0..42bc49cd 100644 --- a/asf_search/CMR/subquery.py +++ b/asf_search/CMR/subquery.py @@ -22,7 +22,7 @@ def build_subqueries(opts: ASFSearchOptions) -> List[ASFSearchOptions]: if params.get('product_list') is not None: params['product_list'] = chunk_list(params['product_list'], CMR_PAGE_SIZE) - list_param_names = ['platform', 'season', 'collections'] # these parameters will dodge the subquery system + list_param_names = ['platform', 'season', 'collections', 'dataset'] # these parameters will dodge the subquery system skip_param_names = ['maxResults']# these params exist in opts, but shouldn't be passed on to subqueries at ALL params = dict([ (k, v) for k, v in params.items() if k not in skip_param_names ]) diff --git a/asf_search/CMR/translate.py b/asf_search/CMR/translate.py index ab74d44a..e38fe193 100644 --- a/asf_search/CMR/translate.py +++ b/asf_search/CMR/translate.py @@ -7,6 +7,7 @@ from shapely.geometry import Polygon from shapely.geometry.base import BaseGeometry from .field_map import field_map +from .datasets import dataset_collections import logging @@ -46,7 +47,20 @@ def translate_opts(opts: ASFSearchOptions) -> list: # If you need to use the temporal key: if any(key in dict_opts for key in ['start', 'end', 'season']): dict_opts = fix_date(dict_opts) - + + if 'dataset' in dict_opts: + if 'collections' not in dict_opts: + dict_opts['collections'] = [] + + for dataset in dict_opts['dataset']: + if collections_by_short_name := dataset_collections.get(dataset): + for concept_ids in collections_by_short_name.values(): + dict_opts['collections'].extend(concept_ids) + else: + raise ValueError(f'Could not find dataset named "{dataset}" provided for dataset keyword.') + + dict_opts.pop('dataset') + # convert the above parameters to a list of key/value tuples cmr_opts = [] for (key, val) in dict_opts.items(): @@ -88,9 +102,16 @@ def translate_opts(opts: ASFSearchOptions) -> list: def should_use_asf_frame(cmr_opts): asf_frame_platforms = ['SENTINEL-1A', 'SENTINEL-1B', 'ALOS'] + asf_frame_datasets = ['SENTINEL-1', 'OPERA-S1', 'SLC-BURST', 'ALOS PALSAR', 'ALOS AVNIR-2'] + + asf_frame_collections = [] + for dataset in asf_frame_datasets: + for concept_ids in dataset_collections.get(dataset).values(): + asf_frame_collections.extend(concept_ids) return any([ p[0] == 'platform[]' and p[1].upper() in asf_frame_platforms + or p[0] == 'echo_collection_id[]' and p[1] in asf_frame_collections for p in cmr_opts ]) diff --git a/asf_search/constants/INTERNAL.py b/asf_search/constants/INTERNAL.py index 8ef3e144..124999dc 100644 --- a/asf_search/constants/INTERNAL.py +++ b/asf_search/constants/INTERNAL.py @@ -1,7 +1,7 @@ ASF_AUTH_HOST = 'auth.asf.alaska.edu' CMR_HOST = 'cmr.earthdata.nasa.gov' -CMR_FORMAT_EXT = 'umm_json_v1_4' +CMR_FORMAT_EXT = 'umm_json' CMR_GRANULE_PATH = f'/search/granules.{CMR_FORMAT_EXT}' CMR_COLLECTIONS = f'/search/collections' CMR_COLLECTIONS_PATH = f'{CMR_COLLECTIONS}.{CMR_FORMAT_EXT}' diff --git a/asf_search/search/search.py b/asf_search/search/search.py index bf7b0529..49f229ee 100644 --- a/asf_search/search/search.py +++ b/asf_search/search/search.py @@ -40,6 +40,7 @@ def search( fullBurstID: Union[str, Iterable[str]] = None, collections: Union[str, Iterable[str]] = None, temporalBaselineDays: Union[str, Iterable[str]] = None, + dataset: Union[str, Iterable[str]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> ASFSearchResults: diff --git a/asf_search/search/search_generator.py b/asf_search/search/search_generator.py index 8763ea7e..e94d3916 100644 --- a/asf_search/search/search_generator.py +++ b/asf_search/search/search_generator.py @@ -54,6 +54,7 @@ def search_generator( fullBurstID: Union[str, Iterable[str]] = None, collections: Union[str, Iterable[str]] = None, temporalBaselineDays: Union[str, Iterable[str]] = None, + dataset: Union[str, Iterable[str]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> Generator[ASFSearchResults, None, None]: diff --git a/tests/Search/test_search.py b/tests/Search/test_search.py index e3d38c80..463d834a 100644 --- a/tests/Search/test_search.py +++ b/tests/Search/test_search.py @@ -1,15 +1,15 @@ from numbers import Number from asf_search import ASFSearchOptions from asf_search.ASFProduct import ASFProduct +from asf_search.CMR.translate import get from asf_search.constants import INTERNAL from asf_search.exceptions import ASFSearchError from asf_search.search import search from asf_search.ASFSearchResults import ASFSearchResults - -import requests - +from asf_search.CMR import dataset_collections from pytest import raises - +from typing import List +import requests import requests_mock def run_test_ASFSearchResults(search_resp): @@ -75,4 +75,19 @@ def custom_matcher(request: requests.Request): with raises(ASFSearchError): results.raise_if_incomplete() - +def run_test_dataset_search(datasets: List): + if any(dataset for dataset in datasets if dataset_collections.get(dataset) is None): + with raises(ValueError): + search(dataset=datasets, maxResults=1) + else: + for dataset in datasets: + valid_shortnames = list(dataset_collections.get(dataset)) + + response = search(dataset=dataset, maxResults=250) + + # Get collection shortName of all granules + shortNames = list(set([shortName for product in response if (shortName:=get(product.umm, 'CollectionReference', 'ShortName')) is not None])) + + # and check that results are limited to the expected datasets by their shortname + for shortName in shortNames: + assert shortName in valid_shortnames diff --git a/tests/pytest-config.yml b/tests/pytest-config.yml index b5dfbe6f..285b9fe3 100644 --- a/tests/pytest-config.yml +++ b/tests/pytest-config.yml @@ -199,6 +199,11 @@ test_types: required_keys: results method: test_output_format +- For running dataset keyword tests: + required_in_title: search-dataset + required_keys: dataset + method: test_search_dataset + - For running jupyter notebook example tests: required_keys: notebook method: test_notebook_examples diff --git a/tests/pytest-managers.py b/tests/pytest-managers.py index 70c58d83..438b1520 100644 --- a/tests/pytest-managers.py +++ b/tests/pytest-managers.py @@ -7,7 +7,7 @@ from ASFSearchResults.test_ASFSearchResults import run_test_output_format, run_test_ASFSearchResults_intersection from ASFSession.test_ASFSession import run_auth_with_cookiejar, run_auth_with_creds, run_auth_with_token, run_test_asf_session_rebuild_auth from BaselineSearch.test_baseline_search import * -from Search.test_search import run_test_ASFSearchResults, run_test_search, run_test_search_http_error +from Search.test_search import run_test_ASFSearchResults, run_test_dataset_search, run_test_search, run_test_search_http_error from Search.test_search_generator import run_test_search_generator, run_test_search_generator_multi from CMR.test_MissionList import run_test_get_project_names @@ -417,6 +417,10 @@ def test_ASFSearchResults_intersection(**kwargs) -> None: wkt = get_resource(kwargs['test_info']['wkt']) run_test_ASFSearchResults_intersection(wkt) +def test_search_dataset(**kwargs) -> None: + dataset = get_resource(kwargs['test_info']['dataset']) + run_test_dataset_search(dataset) + def test_serialization(**args) -> None: test_info = args['test_info'] product = get_resource(test_info.get('product')) diff --git a/tests/yml_tests/test_search.yml b/tests/yml_tests/test_search.yml index 97449d63..2e336bcd 100644 --- a/tests/yml_tests/test_search.yml +++ b/tests/yml_tests/test_search.yml @@ -42,3 +42,15 @@ tests: platform: "Sentinel-1" status_code: 500 report: "Server Error: This is a Test Error" + +- test-search-dataset S1 Datasets: + dataset: ['SENTINEL-1', 'SLC-BURST', 'OPERA-S1'] + +- test-search-dataset S1 Datasets and non-S1: + dataset: ['SENTINEL-1', 'SLC-BURST', 'OPERA-S1', 'UAVSAR'] + +- test-search-dataset fake dataset: + dataset: 'FAKE-DATASET-V1' + +- test-search-dataset S1 Datasets and fake dataset: + dataset: ['SENTINEL-1', 'SLC-BURST', 'OPERA-S1', 'FAKE-DATASET-V2']