diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d7440f..d8dd6ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -66,12 +66,12 @@ repos: args: ['--no-error-on-unmatched-pattern', '--ignore-unknown'] - # Lint: Markdown - - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.39.0 - hooks: - - id: markdownlint - args: ["--fix"] + # Lint: Markdown - Disabled as most rules aren't useful + # - repo: https://github.com/igorshubovych/markdownlint-cli + # rev: v0.39.0 + # hooks: + # - id: markdownlint + # args: ["--fix"] # - repo: https://github.com/asottile/pyupgrade # rev: v3.15.0 diff --git a/src/hazard/docs_store.py b/src/hazard/docs_store.py index 6207291..814bb55 100644 --- a/src/hazard/docs_store.py +++ b/src/hazard/docs_store.py @@ -1,6 +1,6 @@ import json import os -from typing import Any, Dict, Iterable, List, Optional +from typing import Any, Dict, Iterable, List, Optional, cast import s3fs # type: ignore from fsspec import AbstractFileSystem # type: ignore @@ -94,42 +94,47 @@ def write_inventory_stac(self, resources: Iterable[HazardResource]): """Write a hazard models inventory as STAC.""" if self._fs == s3fs.S3FileSystem: - path_Root = self._root + path_root = self._root else: path_root = "." - items = HazardResources(resources=resources).to_stac_items(path_root=path_root, items_as_dicts=True) - for it in items: + items = HazardResources(resources=list(resources)).to_stac_items(path_root=path_root, items_as_dicts=True) + items_cast: List[Dict[str, Any]] = cast(List[Dict[str, Any]], items) + for it in items_cast: with self._fs.open(self._full_path_stac_item(id=it["id"]), "w") as f: f.write(json.dumps(it, indent=4)) catalog_path = self._full_path_stac_catalog() - catalog = self.stac_catalog(items=items) + catalog = self.stac_catalog(items=items_cast) with self._fs.open(catalog_path, "w") as f: json_str = json.dumps(catalog, indent=4) f.write(json_str) collection_path = self._full_path_stac_collection() - collection = self.stac_collection(items=items) + collection = self.stac_collection(items=items_cast) with self._fs.open(collection_path, "w") as f: json_str = json.dumps(collection, indent=4) f.write(json_str) def stac_catalog(self, items: List[Dict[str, Any]]) -> Dict[str, Any]: - + links = [ + {"rel": "self", "href": "./catalog.json"}, + {"rel": "root", "href": "./catalog.json"}, + {"rel": "child", "href": "./collection.json"}, + ] + links.extend([{"rel": "item", "href": f"./{x['id']}.json"} for x in items]) return { "stac_version": "1.0.0", "id": "osc-hazard-indicators-catalog", "type": "Catalog", "description": "OS-C hazard indicators catalog", - "links": [ - {"rel": "self", "href": "./catalog.json"}, - {"rel": "root", "href": "./catalog.json"}, - {"rel": "child", "href": "./collection.json"}, - ] - + [{"rel": "item", "href": f"./{x['id']}.json"} for x in items], + "links": links, } def stac_collection(self, items: List[Dict[str, Any]]) -> Dict[str, Any]: - + links = [ + {"rel": "self", "type": "application/json", "href": "./collection.json"}, + {"rel": "root", "type": "application/json", "href": "./catalog.json"}, + ] + links.extend([{"rel": "item", "href": f"./{x['id']}.json"} for x in items]) return { "stac_version": "1.0.0", "type": "Collection", @@ -143,11 +148,7 @@ def stac_collection(self, items: List[Dict[str, Any]]) -> Dict[str, Any]: "temporal": {"interval": [["1950-01-01T00:00:00Z", "2100-12-31T23:59:59Z"]]}, }, "providers": [{"name": "UKRI", "roles": ["producer"], "url": "https://www.ukri.org/"}], - "links": [ - {"rel": "self", "type": "application/json", "href": "./collection.json"}, - {"rel": "root", "type": "application/json", "href": "./catalog.json"}, - ] - + [{"rel": "item", "href": f"./{x['id']}.json"} for x in items], + "links": links, } def update_inventory(self, resources: Iterable[HazardResource], remove_existing: bool = False): diff --git a/src/hazard/inventory.py b/src/hazard/inventory.py index d316833..a1e1556 100644 --- a/src/hazard/inventory.py +++ b/src/hazard/inventory.py @@ -119,7 +119,7 @@ def to_stac_items(self, path_root: str, items_as_dicts: bool = False) -> List[Un """ keys, values = zip(*self.params.items()) params_permutations = list(itertools.product(*values)) - params_permutations = [dict(zip(keys, p)) for p in params_permutations] + params_permutations_dicts = [dict(zip(keys, p)) for p in params_permutations] scenarios_permutations = [] for s in self.scenarios: @@ -128,7 +128,7 @@ def to_stac_items(self, path_root: str, items_as_dicts: bool = False) -> List[Un permutations = [ dict(**param, **scenario) - for param, scenario in itertools.product(params_permutations, scenarios_permutations) + for param, scenario in itertools.product(params_permutations_dicts, scenarios_permutations) ] items = [] @@ -161,10 +161,12 @@ def to_stac_item( link = pystac.Link(rel="collection", media_type="application/json", target="./collection.json") + coordinates = self.map.bounds if self.map else None + bbox = self.map.bbox if self.map else None stac_item = pystac.Item( id=item_id, - geometry={"type": "Polygon", "coordinates": [self.map.bounds]}, - bbox=self.map.bbox, + geometry={"type": "Polygon", "coordinates": [coordinates]}, + bbox=bbox, datetime=None, start_datetime=datetime.datetime(2000, 1, 1, tzinfo=datetime.timezone.utc), end_datetime=datetime.datetime(2100, 1, 1, tzinfo=datetime.timezone.utc), @@ -186,7 +188,7 @@ def to_stac_item( class HazardResources(BaseModel): resources: List[HazardResource] - def to_stac_items(self, path_root: str, items_as_dicts: bool = False) -> List[Dict[str, Any]]: + def to_stac_items(self, path_root: str, items_as_dicts: bool = False) -> List[Union[pystac.Item, Dict]]: """ converts hazard resources to a list of STAC items. """ diff --git a/src/hazard/models/degree_days.py b/src/hazard/models/degree_days.py index 853a68c..6415846 100644 --- a/src/hazard/models/degree_days.py +++ b/src/hazard/models/degree_days.py @@ -265,20 +265,22 @@ def _calculate_single_year_indicators(self, source: OpenDataset, item: BatchItem above = self._degree_days_indicator(tas, year, AboveBelow.ABOVE) below = self._degree_days_indicator(tas, year, AboveBelow.BELOW) logger.info(f"Calculation complete for year {year}") + bounds_above = self.resources["above"].map.bounds if self.resources["above"].map else [] + bounds_below = self.resources["below"].map.bounds if self.resources["below"].map else [] return [ Indicator( above, PurePosixPath( self.resources["above"].path.format(gcm=item.gcm, scenario=item.scenario, year=item.central_year) ), - self.resources["above"].map.bounds, + bounds_above, ), Indicator( below, PurePosixPath( self.resources["below"].path.format(gcm=item.gcm, scenario=item.scenario, year=item.central_year) ), - self.resources["below"].map.bounds, + bounds_below, ), ] @@ -307,7 +309,8 @@ def _degree_days_indicator(self, tas: xr.DataArray, year: int, above_below: Abov ) return da - def _resource(self): + def _resource(self) -> Dict[str, HazardResource]: # type: ignore[override] + # Unsure why this returns a dict vs a single resource resources: Dict[str, HazardResource] = {} for above_below in ["above", "below"]: with open(os.path.join(os.path.dirname(__file__), "degree_days.md"), "r") as f: diff --git a/src/hazard/models/drought_index.py b/src/hazard/models/drought_index.py index 29cd6a8..083092a 100644 --- a/src/hazard/models/drought_index.py +++ b/src/hazard/models/drought_index.py @@ -36,7 +36,7 @@ class BatchItem: class ZarrWorkingStore(Protocol): - def get_store(self, path: str): ... + def get_store(self, path: str): ... # noqa:E704 class S3ZarrWorkingStore(ZarrWorkingStore): @@ -376,7 +376,7 @@ def _resource(self) -> HazardResource: description="", display_groups=["Drought SPEI index"], # display names of groupings group_id="", - map=MapInfo( + map=MapInfo( # type: ignore[call-arg] # has a default value for bbox colormap=Colormap( name="heating", nodata_index=0, diff --git a/src/hazard/models/multi_year_average.py b/src/hazard/models/multi_year_average.py index 4b72ef1..e7bba22 100644 --- a/src/hazard/models/multi_year_average.py +++ b/src/hazard/models/multi_year_average.py @@ -204,4 +204,4 @@ def _get_indicators(self, item: BatchItem, data_arrays: List[xr.DataArray], para ] @abstractmethod - def _resource(self) -> HazardResource: ... + def _resource(self) -> HazardResource: ... # noqa:E704 diff --git a/src/hazard/models/water_temp.py b/src/hazard/models/water_temp.py index 7d02738..4498af1 100644 --- a/src/hazard/models/water_temp.py +++ b/src/hazard/models/water_temp.py @@ -85,8 +85,7 @@ def water_temp_download_path(self, gcm: str, scenario: str, year: int) -> Tuple[ filename = self.filename(gcm, scenario, year) url = ( "https://geo.public.data.uu.nl/vault-futurestreams/research-futurestreams%5B1633685642%5D/" - + f"original/waterTemp/{adjusted_scenario}/{adjusted_gcm}/" - + filename + f"original/waterTemp/{adjusted_scenario}/{adjusted_gcm}/{filename}" ) return os.path.join(self.working_dir, filename), url @@ -273,7 +272,7 @@ def _resource(self) -> HazardResource: description=description, display_groups=["Weeks with average water temperature above threshold in °C"], # display names of groupings group_id="", - map=MapInfo( + map=MapInfo( # type: ignore[call-arg] # has a default value for bbox colormap=Colormap( name="heating", nodata_index=0, @@ -301,7 +300,7 @@ def _resource(self) -> HazardResource: def _other_resource(self) -> HazardResource: assert self.resource.map is not None - map = MapInfo( + map = MapInfo( # type: ignore[call-arg] # has a default value for bbox colormap=self.resource.map.colormap, bounds=self.resource.map.bounds, path=self.resource.map.path.format(gcm="E2O", scenario="{scenario}", year="{year}"), diff --git a/src/hazard/models/wet_bulb_globe_temp.py b/src/hazard/models/wet_bulb_globe_temp.py index baa3946..86a1ecb 100644 --- a/src/hazard/models/wet_bulb_globe_temp.py +++ b/src/hazard/models/wet_bulb_globe_temp.py @@ -119,7 +119,7 @@ def _resource(self) -> HazardResource: description=description, display_groups=["Days with wet-bulb globe temperature above threshold in °C"], # display names of groupings group_id="", - map=MapInfo( + map=MapInfo( # type: ignore[call-arg] # has a default value for bbox colormap=Colormap( name="heating", nodata_index=0, diff --git a/src/hazard/models/work_loss.py b/src/hazard/models/work_loss.py index 8341eea..2c8fcf8 100644 --- a/src/hazard/models/work_loss.py +++ b/src/hazard/models/work_loss.py @@ -78,7 +78,7 @@ def _resource(self) -> HazardResource: display_groups=["Mean work loss"], # display names of groupings # we want "Mean work loss" -> "Low intensity", "Medium intensity", "High intensity" -> "GCM1", "GCM2", ... group_id="", - map=MapInfo( + map=MapInfo( # type: ignore[call-arg] # has a default value for bbox colormap=Colormap( name="heating", nodata_index=0, diff --git a/src/hazard/onboard/iris_wind.py b/src/hazard/onboard/iris_wind.py index 21ab1d6..9b769bd 100644 --- a/src/hazard/onboard/iris_wind.py +++ b/src/hazard/onboard/iris_wind.py @@ -98,7 +98,7 @@ def _hazard_resource(self) -> HazardResource: description=description, group_id="iris_osc", display_groups=[], - map=MapInfo( + map=MapInfo( # type: ignore[call-arg] # has a default value for bbox bounds=[(-180.0, 60.0), (180.0, 60.0), (180.0, -60.0), (-180.0, -60.0)], colormap=Colormap( name="heating", diff --git a/src/hazard/onboard/jupiter.py b/src/hazard/onboard/jupiter.py index 7e168c2..fd90203 100644 --- a/src/hazard/onboard/jupiter.py +++ b/src/hazard/onboard/jupiter.py @@ -127,10 +127,10 @@ def inventory(self) -> Iterable[HazardResource]: a given location based on several parameters from multiple bias corrected and downscaled Global Climate Models (GCMs). For example, if the probability of occurrence of a wildfire is 5% in July, 20% in August, 10% in September and 0% for other months, the hazard indicator value is 20%. - """, + """, # noqa:W503 group_id="jupiter_osc", display_groups=[], - map=MapInfo( + map=MapInfo( # type: ignore[call-arg] # has a default value for bbox bounds=[ (-180.0, 85.0), (180.0, 85.0), @@ -175,10 +175,10 @@ def inventory(self) -> Iterable[HazardResource]: This drought model computes the number of months per annum where the 3-month rolling average of SPEI is below -2 based on the mean values of several parameters from bias-corrected and downscaled multiple Global Climate Models (GCMs). - """, + """, # noqa:W503 group_id="jupiter_osc", display_groups=[], - map=MapInfo( + map=MapInfo( # type: ignore[call-arg] # has a default value for bbox bounds=[ (-180.0, 85.0), (180.0, 85.0), @@ -217,10 +217,10 @@ def inventory(self) -> Iterable[HazardResource]: This model computes the maximum daily water equivalent precipitation (in mm) measured at the 100 year return period based on the mean of the precipitation distribution from multiple bias corrected and downscaled Global Climate Models (GCMs). - """, + """, # noqa:W503 group_id="jupiter_osc", display_groups=[], - map=MapInfo( + map=MapInfo( # type: ignore[call-arg] # has a default value for bbox bounds=[ (-180.0, 85.0), (180.0, 85.0), @@ -259,10 +259,10 @@ def inventory(self) -> Iterable[HazardResource]: This hail model computes the number of days per annum where hail exceeding 5 cm diameter is possible based on the mean distribution of several parameters across multiple bias-corrected and downscaled Global Climate Models (GCMs). - """, + """, # noqa:W503 group_id="jupiter_osc", display_groups=[], - map=MapInfo( + map=MapInfo( # type: ignore[call-arg] # has a default value for bbox bounds=[ (-180.0, 85.0), (180.0, 85.0), @@ -301,10 +301,10 @@ def inventory(self) -> Iterable[HazardResource]: This heat model computes the number of days exceeding 35°C per annum based on the mean of distribution fits to the bias-corrected and downscaled high temperature distribution across multiple Global Climate Models (GCMs). - """, + """, # noqa:W503 group_id="jupiter_osc", display_groups=[], - map=MapInfo( + map=MapInfo( # type: ignore[call-arg] # has a default value for bbox bounds=[ (-180.0, 85.0), (180.0, 85.0), @@ -343,10 +343,10 @@ def inventory(self) -> Iterable[HazardResource]: This wind speed model computes the maximum 1-minute sustained wind speed (in km/hr) experienced over a 100 year return period based on mean wind speed distributions from multiple Global Climate Models (GCMs). - """, + """, # noqa:W503 group_id="jupiter_osc", display_groups=[], - map=MapInfo( + map=MapInfo( # type: ignore[call-arg] # has a default value for bbox bounds=[ (-180.0, 85.0), (180.0, 85.0), @@ -387,10 +387,10 @@ def inventory(self) -> Iterable[HazardResource]: cells within the 30-km cell that have non-zero flooding at that return period. This model uses a 30-km grid that experiences flooding at the 200-year return period. Open oceans are excluded. - """, + """, # noqa:W503 group_id="jupiter_osc", display_groups=[], - map=MapInfo( + map=MapInfo( # type: ignore[call-arg] # has a default value for bbox bounds=[ (-180.0, 85.0), (180.0, 85.0), diff --git a/src/hazard/onboard/tudelft_flood.py b/src/hazard/onboard/tudelft_flood.py index 9a061b5..0854ee2 100644 --- a/src/hazard/onboard/tudelft_flood.py +++ b/src/hazard/onboard/tudelft_flood.py @@ -188,7 +188,7 @@ def inventory(self) -> Iterable[HazardResource]: description=description, group_id="", display_groups=[], - map=MapInfo( + map=MapInfo( # type: ignore[call-arg] # has a default value for bbox bounds=[], colormap=Colormap( max_index=255, diff --git a/src/hazard/onboard/wri_aqueduct_flood.py b/src/hazard/onboard/wri_aqueduct_flood.py index 03c466b..083cf56 100644 --- a/src/hazard/onboard/wri_aqueduct_flood.py +++ b/src/hazard/onboard/wri_aqueduct_flood.py @@ -51,7 +51,7 @@ def batch_items(self) -> Iterable[BatchItem]: # "inundation/wri/v2/inuncoast_historical_wtsub_hist_0"]] return items - def batch_items_riverine(self): + def batch_items_riverine(self) -> List[BatchItem]: gcms = [ "00000NorESM1-M", "0000GFDL-ESM2M", @@ -81,7 +81,7 @@ def batch_items_riverine(self): items.append(BatchItem(self._resource(path), path, scenario, str(year), filename_return_period)) return items - def batch_items_coastal(self): + def batch_items_coastal(self) -> List[BatchItem]: models = ["0", "0_perc_05", "0_perc_50"] subs = ["wtsub", "nosub"] years = [2030, 2050, 2080] @@ -103,14 +103,15 @@ def batch_items_coastal(self): ) # plus two extra historical/baseline items for sub in subs: - scenario, year = "historical", "hist" - path, filename_return_period = self.path_coastal(scenario, sub, year, "0") + hist_scenario: str = "historical" + hist_year: str = "hist" + path, filename_return_period = self.path_coastal(hist_scenario, sub, hist_year, "0") items.append( BatchItem( self._resource(path), path, - scenario, - str(year), + hist_scenario, + hist_year, filename_return_period, ) ) @@ -153,7 +154,7 @@ def inventory(self) -> Iterable[HazardResource]: World Resources Institute Aqueduct Floods baseline riverine model using historical data. """ - + aqueduct_description, + + aqueduct_description, # noqa:W503 "map": { "colormap": wri_colormap, "path": "inundation/wri/v2/inunriver_{scenario}_000000000WATCH_{year}_map", @@ -179,7 +180,7 @@ def inventory(self) -> Iterable[HazardResource]: Bjerknes Centre for Climate Research, Norwegian Meteorological Institute. """ - + aqueduct_description, + + aqueduct_description, # noqa:W503 "map": { "colormap": wri_colormap, "index_values": [8], @@ -203,7 +204,7 @@ def inventory(self) -> Iterable[HazardResource]: Geophysical Fluid Dynamics Laboratory (NOAA). """ - + aqueduct_description, + + aqueduct_description, # noqa:W503 "map": { "colormap": wri_colormap, "index_values": [8], @@ -227,7 +228,7 @@ def inventory(self) -> Iterable[HazardResource]: Met Office Hadley Centre. """ - + aqueduct_description, + + aqueduct_description, # noqa:W503 "map": { "colormap": wri_colormap, "index_values": [8], @@ -251,7 +252,7 @@ def inventory(self) -> Iterable[HazardResource]: Institut Pierre Simon Laplace """ - + aqueduct_description, + + aqueduct_description, # noqa:W503 "map": { "colormap": wri_colormap, "index_values": [8], @@ -276,7 +277,7 @@ def inventory(self) -> Iterable[HazardResource]: for Marine-Earth Science and Technology. """ - + aqueduct_description, + + aqueduct_description, # noqa:W503 "map": { "colormap": wri_colormap, "index_values": [8], @@ -311,7 +312,7 @@ def inventory(self) -> Iterable[HazardResource]: World Resources Institute Aqueduct Floods baseline coastal model using historical data. Model excludes subsidence. """ - + aqueduct_description, + + aqueduct_description, # noqa:W503 "map": { "colormap": wri_colormap, "index_values": [8], @@ -333,7 +334,7 @@ def inventory(self) -> Iterable[HazardResource]: World Resource Institute Aqueduct Floods coastal model, excluding subsidence; 95th percentile sea level rise. """ - + aqueduct_description, + + aqueduct_description, # noqa:W503 "map": { "colormap": wri_colormap, "index_values": [8], @@ -357,7 +358,7 @@ def inventory(self) -> Iterable[HazardResource]: World Resource Institute Aqueduct Floods coastal model, excluding subsidence; 5th percentile sea level rise. """ - + aqueduct_description, + + aqueduct_description, # noqa:W503 "map": { "colormap": wri_colormap, "index_values": [8], @@ -381,7 +382,7 @@ def inventory(self) -> Iterable[HazardResource]: World Resource Institute Aqueduct Floods model, excluding subsidence; 50th percentile sea level rise. """ - + aqueduct_description, + + aqueduct_description, # noqa:W503 "map": { "colormap": wri_colormap, "index_values": [8], @@ -405,7 +406,7 @@ def inventory(self) -> Iterable[HazardResource]: World Resource Institute Aqueduct Floods model, excluding subsidence; baseline (based on historical data). """ - + aqueduct_description, + + aqueduct_description, # noqa:W503 "map": { "colormap": wri_colormap, "index_values": [8], @@ -426,7 +427,7 @@ def inventory(self) -> Iterable[HazardResource]: World Resource Institute Aqueduct Floods model, including subsidence; 95th percentile sea level rise. """ - + aqueduct_description, + + aqueduct_description, # noqa:W503 "map": { "colormap": wri_colormap, "index_values": [8], @@ -450,7 +451,7 @@ def inventory(self) -> Iterable[HazardResource]: World Resource Institute Aqueduct Floods model, including subsidence; 5th percentile sea level rise. """ - + aqueduct_description, + + aqueduct_description, # noqa:W503 "map": { "colormap": wri_colormap, "index_values": [8], @@ -474,7 +475,7 @@ def inventory(self) -> Iterable[HazardResource]: World Resource Institute Aqueduct Floods model, including subsidence; 50th percentile sea level rise. """ - + aqueduct_description, + + aqueduct_description, # noqa:W503 "map": { "colormap": wri_colormap, "index_values": [8], diff --git a/src/hazard/protocols.py b/src/hazard/protocols.py index d335071..76a27c8 100644 --- a/src/hazard/protocols.py +++ b/src/hazard/protocols.py @@ -13,7 +13,7 @@ class Averageable(Protocol): class OpenDataset(Protocol): """Open XArray Dataset for Global Circulation Model (GCM), scenario and quantity for whole year specified.""" - def gcms(self) -> Iterable[str]: ... + def gcms(self) -> Iterable[str]: ... # noqa:E704 def open_dataset_year( self, gcm: str, scenario: str, quantity: str, year: int, chunks=None @@ -24,13 +24,13 @@ def open_dataset_year( class ReadDataArray(Protocol): """Read DataArray.""" - def read(self, path: str) -> xr.DataArray: ... + def read(self, path: str) -> xr.DataArray: ... # noqa:E704 class WriteDataArray(Protocol): """Write DataArray.""" - def write( + def write( # noqa:E704 self, path: str, data_array: xr.DataArray, @@ -45,15 +45,15 @@ class ReadWriteDataArray(ReadDataArray, WriteDataArray): ... # noqa: E701 class WriteDataset(Protocol): """Write DataArray.""" - def write(self, path: str, dataset: xr.Dataset): ... + def write(self, path: str, dataset: xr.Dataset): ... # noqa:E704 T = typing.TypeVar("T") class PTransform(Protocol): - def batch_items(self) -> Iterable[T]: ... + def batch_items(self) -> Iterable[T]: ... # noqa:E704 - def process_item(self, item: T) -> xr.DataArray: ... + def process_item(self, item: T) -> xr.DataArray: ... # noqa:E704 - def item_path(self, item: T) -> str: ... + def item_path(self, item: T) -> str: ... # noqa:E704 diff --git a/src/hazard/sources/__init__.py b/src/hazard/sources/__init__.py index df75bef..43dabd2 100644 --- a/src/hazard/sources/__init__.py +++ b/src/hazard/sources/__init__.py @@ -1,9 +1,8 @@ -from typing import Literal, Mapping, Dict, Any, Protocol, Type, Callable +from typing import Any, Callable, Dict, Literal, Mapping from hazard.protocols import OpenDataset from hazard.sources.nex_gddp_cmip6 import NexGddpCmip6 from hazard.sources.ukcp18_rcp85 import Ukcp18Rcp85 -from hazard.sources.wri_aqueduct import WRIAqueductSource SourceDataset = Literal[ "NEX-GDDP-CMIP6", diff --git a/src/hazard/sources/nex_gddp_cmip6.py b/src/hazard/sources/nex_gddp_cmip6.py index 652c598..895ad1c 100644 --- a/src/hazard/sources/nex_gddp_cmip6.py +++ b/src/hazard/sources/nex_gddp_cmip6.py @@ -65,7 +65,7 @@ def path(self, gcm="NorESM2-MM", scenario="ssp585", quantity="tas", year=2030): self.root, f"NEX-GDDP-CMIP6/{gcm}/{scenario}/{variant_label}/{quantity}/", ) - + filename, + + filename, # noqa:W503 filename, ) diff --git a/src/hazard/utilities/map_utilities.py b/src/hazard/utilities/map_utilities.py index ee1b758..b8b5b98 100644 --- a/src/hazard/utilities/map_utilities.py +++ b/src/hazard/utilities/map_utilities.py @@ -85,7 +85,7 @@ def transform_epsg4326_to_epsg3857(src: xr.DataArray): return reprojected -def highest_zoom_slippy_maps(src: xr.DataArray): ... +def highest_zoom_slippy_maps(src: xr.DataArray): ... # noqa:E704 def check_map_bounds(da: xr.DataArray): diff --git a/src/hazard/utilities/s3_utilities.py b/src/hazard/utilities/s3_utilities.py index eb8d790..f3aa2a4 100644 --- a/src/hazard/utilities/s3_utilities.py +++ b/src/hazard/utilities/s3_utilities.py @@ -256,7 +256,7 @@ def sync_buckets( different = set(key for key in all_diffs if key not in missing) logger.info( f"Copying {len(missing)} missing files from {source_bucket_name} to {target_bucket_name}: " - + _first_5_last_5(list(missing)) + + _first_5_last_5(list(missing)) # noqa:W503 ) if not dry_run: copy_objects( @@ -268,7 +268,7 @@ def sync_buckets( ) logger.info( f"Copying {len(different)} different files from {source_bucket_name} to {target_bucket_name}: " - + _first_5_last_5(list(different)) + + _first_5_last_5(list(different)) # noqa:W503 ) if not dry_run: copy_objects( diff --git a/src/hazard/utilities/zarr_utilities.py b/src/hazard/utilities/zarr_utilities.py index bbc9ec4..f8b3931 100644 --- a/src/hazard/utilities/zarr_utilities.py +++ b/src/hazard/utilities/zarr_utilities.py @@ -1,124 +1,11 @@ -import logging import os import pathlib -import sys -from typing import List, Tuple -import numpy as np -import s3fs # type: ignore -import zarr # type: ignore -from affine import Affine # type: ignore from dotenv import load_dotenv -def add_logging_output_to_stdout(log): - handler = logging.StreamHandler(sys.stdout) - handler.setLevel(logging.DEBUG) - formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") - handler.setFormatter(formatter) - log.addHandler(handler) - - -def get_coordinates(longitudes, latitudes, transform): - coords = np.vstack((longitudes, latitudes, np.ones(len(longitudes)))) - inv_trans = ~transform - mat = np.array(inv_trans).reshape(3, 3) - frac_image_coords = mat @ coords - image_coords = np.floor(frac_image_coords).astype(int) - return image_coords - - -def get_geotiff_meta_data(path, s3): - from tifffile.tifffile import TiffFile # type: ignore - - with s3.open(path) as f: - with TiffFile(f) as tif: - scale: Tuple[float, float, float] = tif.geotiff_metadata["ModelPixelScale"] - tie_point: List[float] = tif.geotiff_metadata["ModelTiepoint"] - shape: List[int] = tif.series[0].shape - i, j, k, x, y, z = tie_point[0:6] - sx, sy, sz = scale - transform = Affine(sx, 0.0, x - i * sx, 0.0, -sy, y + j * sy) - return shape, transform - - def set_credential_env_variables(): dotenv_dir = os.environ.get("CREDENTIAL_DOTENV_DIR", os.getcwd()) dotenv_path = pathlib.Path(dotenv_dir) / "credentials.env" if os.path.exists(dotenv_path): load_dotenv(dotenv_path=dotenv_path, override=True) - - -def zarr_create(group_path, path, s3, shape, transform, return_periods): - """ - Create zarr with given shape and affine transform - """ - store = s3fs.S3Map(root=group_path, s3=s3, check=False) - root = zarr.group(store=store) # open group as root - - z = root.create_dataset( - path, - shape=( - 1 if return_periods is None else len(return_periods), - shape[0], - shape[1], - ), - chunks=(1 if return_periods is None else len(return_periods), 1000, 1000), - dtype="f4", - overwrite=True, - ) # path interpreted as path within group - trans_members = [ - transform.a, - transform.b, - transform.c, - transform.d, - transform.e, - transform.f, - ] - mat3x3 = [x * 1.0 for x in trans_members] + [0.0, 0.0, 1.0] - z.attrs["transform_mat3x3"] = mat3x3 - if return_periods is not None: - z.attrs["index_values"] = return_periods - z.attrs["index_name"] = "return period (years)" - return z - - -def zarr_get_transform(zarr_array): - t = zarr_array.attrs["transform_mat3x3"] # type: ignore - transform = Affine(t[0], t[1], t[2], t[3], t[4], t[5]) - return transform - - -def zarr_remove(group_path, path, s3): - """ - Remove zarr array - """ - store = s3fs.S3Map(root=group_path, s3=s3, check=False) - root = zarr.open(store=store, mode="w") # open group as root - root.pop(path) - - -def zarr_read(group_path, path, s3, index): - """ - Read data and transform from zarr - """ - store = s3fs.S3Map(root=group_path, s3=s3, check=False) - root = zarr.open(store, mode="r") - z = root[path] - t = z.attrs["transform_mat3x3"] # type: ignore - transform = Affine(t[0], t[1], t[2], t[3], t[4], t[5]) - return z[index, :, :], transform - - -def zarr_write(src_path, src_s3, dest_zarr, index): - """ - Writes data from GeoTiff sepecified by src_path and src_s3 S3FileSystem to - destination zarr array dest_zarr, putting data into index. - """ - from tifffile.tifffile import TiffFile - - with src_s3.open(src_path) as f: - with TiffFile(f) as tif: - store = tif.series[0].aszarr() - z_in = zarr.open(store, mode="r") - dest_zarr[index, :, :] = z_in[:, :] diff --git a/tests/conftest.py b/tests/conftest.py index ea1cdc6..78dac78 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,14 +1,13 @@ import os from datetime import datetime -from typing import Dict, Optional, Tuple +from typing import Dict, Iterable, List, Optional, Tuple import numpy as np import pandas as pd # type: ignore import pytest import xarray as xr -from pytest import approx -from hazard.protocols import OpenDataset, WriteDataset +from hazard.protocols import OpenDataset, ReadWriteDataArray from hazard.utilities import zarr_utilities @@ -34,21 +33,31 @@ def test_output_dir(): class TestSource(OpenDataset): """Mocked source for testing.""" - def __init__(self, datasets: Dict[Tuple[str, int], xr.Dataset]): + def __init__(self, datasets: Dict[Tuple[str, int], xr.Dataset], gcms: Iterable[str]): self.datasets = datasets + self._gcms = gcms + + def gcms(self) -> Iterable[str]: + return self._gcms def open_dataset_year(self, gcm: str, scenario: str, quantity: str, year: int, chunks=None) -> xr.Dataset: return self.datasets[(quantity, year)] # ignore scenario and gcm: we test just a single one -class TestTarget(WriteDataset): +class TestTarget(ReadWriteDataArray): """Mocked target for testing.""" def __init__(self): self.datasets = {} - def write(self, path: str, dataset: xr.Dataset, spatial_coords: Optional[bool] = False): - self.datasets[path] = dataset + def write( # noqa:E704 + self, + path: str, + data_array: xr.DataArray, + chunks: Optional[List[int]] = None, + spatial_coords: Optional[bool] = True, + ): + self.datasets[path] = data_array def read(self, path: str): return self.datasets[path].rename({"lat": "latitude", "lon": "longitude"}) diff --git a/tests/test_drought_indicators.py b/tests/test_drought_indicators.py index 0cfae46..588249f 100644 --- a/tests/test_drought_indicators.py +++ b/tests/test_drought_indicators.py @@ -10,7 +10,6 @@ from hazard.docs_store import DocStore from hazard.models.drought_index import DroughtIndicator, LocalZarrWorkingStore, ProgressStore, S3ZarrWorkingStore -from tests.conftest import TestTarget @pytest.mark.skip(reason="incomplete") diff --git a/tests/test_heat_indicators.py b/tests/test_heat_indicators.py index 52c31c8..75fecdb 100644 --- a/tests/test_heat_indicators.py +++ b/tests/test_heat_indicators.py @@ -1,10 +1,8 @@ -import json import os -from typing import Dict, List +from typing import List import fsspec.implementations.local as local # type: ignore import numpy as np -import pandas as pd # type: ignore import pytest import s3fs # type: ignore import xarray as xr @@ -13,10 +11,9 @@ from pytest import approx import hazard.utilities.zarr_utilities as zarr_utilities -from hazard.docs_store import DocStore, HazardResources # type: ignore +from hazard.docs_store import DocStore from hazard.models.degree_days import BatchItem, DegreeDays, HeatingCoolingDegreeDays from hazard.models.work_loss import WorkLossIndicator -from hazard.protocols import OpenDataset, WriteDataset from hazard.sources.nex_gddp_cmip6 import NexGddpCmip6 from hazard.sources.osc_zarr import OscZarr from tests.conftest import TestSource, TestTarget, _create_test_dataset_averaged, _create_test_datasets_tas @@ -29,7 +26,7 @@ def test_degree_days_mocked(): gcm = "NorESM2-MM" scenario = "ssp585" year = 2030 - source = TestSource(_create_test_datasets_tas()) + source = TestSource(_create_test_datasets_tas(), [gcm]) target = TestTarget() # cut down the transform model = DegreeDays(window_years=2, gcms=[gcm], scenarios=[scenario], central_years=[year]) @@ -45,7 +42,7 @@ def test_degree_days_mocked(): ) -def test_work_loss_mocked(): +def test_work_loss_mocked() -> None: """Test degree days calculation based on mocked data.""" gcm = "NorESM2-MM" scenario = "ssp585" @@ -55,7 +52,7 @@ def test_work_loss_mocked(): alpha_light = (32.98, 17.81) alpha_medium = (30.94, 16.64) alpha_heavy = (24.64, 22.72) - source = TestSource(test_sets) + source = TestSource(test_sets, [gcm]) target = TestTarget() # cut down the transform model = WorkLossIndicator(window_years=2, gcms=[gcm], scenarios=[scenario], central_years=[year]) diff --git a/tests/test_inventory.py b/tests/test_inventory.py index eeb1b53..7c8c7f2 100644 --- a/tests/test_inventory.py +++ b/tests/test_inventory.py @@ -3,7 +3,7 @@ import fsspec.implementations.local as local # type: ignore import pytest -from hazard.docs_store import DocStore, HazardResources +from hazard.docs_store import DocStore from hazard.models.days_tas_above import DaysTasAboveIndicator from hazard.models.degree_days import DegreeDays, HeatingCoolingDegreeDays from hazard.models.drought_index import DroughtIndicator @@ -17,7 +17,7 @@ from hazard.onboard.wri_aqueduct_water_risk import WRIAqueductWaterRisk from hazard.utilities import zarr_utilities # type: ignore -from .conftest import test_output_dir +from .conftest import test_output_dir # noqa:F401 - Used as it's a fixture def test_create_inventory(test_output_dir): # noqa: F811 diff --git a/tests/test_onboarding.py b/tests/test_onboarding.py index 408c14a..eead7e7 100644 --- a/tests/test_onboarding.py +++ b/tests/test_onboarding.py @@ -1,6 +1,5 @@ import logging import os -import pathlib import sys from pathlib import PurePosixPath @@ -26,9 +25,7 @@ ) from hazard.sources.nex_gddp_cmip6 import NexGddpCmip6 from hazard.sources.osc_zarr import OscZarr -from hazard.sources.wri_aqueduct import WRIAqueductSource from hazard.utilities import s3_utilities, zarr_utilities -from hazard.utilities.tiles import create_tile_set @pytest.fixture diff --git a/tests/test_temperature_indicators.py b/tests/test_temperature_indicators.py index e3ad474..78241d0 100644 --- a/tests/test_temperature_indicators.py +++ b/tests/test_temperature_indicators.py @@ -1,5 +1,4 @@ import os -from typing import List import fsspec.implementations.local as local # type: ignore import numpy as np @@ -13,7 +12,13 @@ from hazard.sources.nex_gddp_cmip6 import NexGddpCmip6 from hazard.sources.osc_zarr import OscZarr # type: ignore -from .conftest import TestSource, TestTarget, _create_test_datasets_hurs, _create_test_datasets_tas, test_output_dir +from .conftest import ( # noqa:F401 - Used as it's a fixture + TestSource, + TestTarget, + _create_test_datasets_hurs, + _create_test_datasets_tas, + test_output_dir, +) def test_days_tas_above_mocked(): @@ -21,7 +26,7 @@ def test_days_tas_above_mocked(): gcm = "NorESM2-MM" scenario = "ssp585" year = 2030 - source = TestSource(_create_test_datasets_tas("tas")) + source = TestSource(_create_test_datasets_tas("tas"), [gcm]) target = TestTarget() # cut down the transform model = DaysTasAboveIndicator( @@ -51,7 +56,7 @@ def test_days_wbgt_above_mocked(): test_sets = _create_test_datasets_tas(quantity="tas") test_sets.update(_create_test_datasets_hurs()) threshold_temps_c = 27.0 - source = TestSource(test_sets) + source = TestSource(test_sets, [gcm]) target = TestTarget() # cut down the transform model = WetBulbGlobeTemperatureAboveIndicator( @@ -65,7 +70,6 @@ def test_days_wbgt_above_mocked(): result = target.datasets[ "chronic_heat/osc/v2/days_wbgt_above_{gcm}_{scenario}_{year}".format(gcm=gcm, scenario=scenario, year=year) ] - expected: List[xr.DataArray] = [] with source.open_dataset_year(gcm, scenario, "tas", 2029).tas as t0: with source.open_dataset_year(gcm, scenario, "hurs", 2029).hurs as h0: tas_c = t0 - 273.15 # convert from K to C diff --git a/tests/test_tile_creation.py b/tests/test_tile_creation.py index d4f11f6..d52d349 100644 --- a/tests/test_tile_creation.py +++ b/tests/test_tile_creation.py @@ -2,7 +2,6 @@ from sys import stdout from typing import List -import dask import numpy as np import pytest import rasterio.transform @@ -12,12 +11,7 @@ from rasterio.warp import Resampling from hazard.indicator_model import IndicatorModel -from hazard.models.days_tas_above import DaysTasAboveIndicator -from hazard.models.degree_days import DegreeDays -from hazard.models.work_loss import WorkLossIndicator -from hazard.onboard.jupiter import Jupiter from hazard.onboard.tudelft_flood import TUDelftRiverFlood -from hazard.onboard.wri_aqueduct_flood import WRIAqueductFlood from hazard.sources.osc_zarr import OscZarr from hazard.utilities import zarr_utilities from hazard.utilities.tiles import create_tile_set, create_tiles_for_resource @@ -25,7 +19,7 @@ from .conftest import test_output_dir # noqa: F401 -def test_convert_tiles_mocked(test_output_dir): +def test_convert_tiles_mocked(test_output_dir): # noqa: F811 not unused, its a fixture """We are combining useful logic from a few sources. rio_tiler and titiler are very useful and also: https://github.com/mapbox/rio-mbtiles @@ -80,13 +74,13 @@ def test_convert_tiles_mocked(test_output_dir): @pytest.mark.skip(reason="Example not test") -def test_map_tiles_from_model(test_output_dir): # noqa: F811 +def test_map_tiles_from_model(test_output_dir) -> None: # noqa: F811 local_store = zarr.DirectoryStore(os.path.join(test_output_dir, "hazard", "hazard.zarr")) source = OscZarr(store=local_store) target = source models: List[IndicatorModel] = [ - TUDelftRiverFlood(None), + TUDelftRiverFlood("None"), # WRIAqueductFlood(), # DegreeDays(), # Jupiter(), diff --git a/tests/test_utilities.py b/tests/test_utilities.py index 3638ef8..1c7569b 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -2,13 +2,11 @@ import numpy as np # type: ignore import zarr # type: ignore -from affine import Affine -from hazard.onboard.storm_wind import BatchItem, STORMIndicator # type: ignore from hazard.sources.osc_zarr import OscZarr from hazard.utilities.xarray_utilities import data_array, empty_data_array, global_crs_transform -from .conftest import test_output_dir +from .conftest import test_output_dir # noqa:F401 used, it's a fixture def test_xarray_write_small(test_output_dir): # noqa: F811 diff --git a/tests/test_water_temp_indicators.py b/tests/test_water_temp_indicators.py index e881862..5f394b3 100644 --- a/tests/test_water_temp_indicators.py +++ b/tests/test_water_temp_indicators.py @@ -1,5 +1,4 @@ import numpy as np -import pytest import xarray as xr from hazard.models.water_temp import FutureStreamsSource, WaterTemperatureAboveIndicator @@ -13,14 +12,14 @@ def test_future_streams_source(): _, url = source.water_temp_download_path("E2O", "historical", 1995) assert ( url - == "https://geo.public.data.uu.nl/vault-futurestreams/research-futurestreams%5B1633685642%5D/original/" - + "waterTemp/hist/E2O/waterTemp_weekAvg_output_E2O_hist_1986-01-07_to_1995-12-30.nc" + == "https://geo.public.data.uu.nl/vault-futurestreams/research-futurestreams%5B1633685642%5D/original/" # noqa:W503 + + "waterTemp/hist/E2O/waterTemp_weekAvg_output_E2O_hist_1986-01-07_to_1995-12-30.nc" # noqa:W503 ) _, url = source.water_temp_download_path("NorESM", "rcp8p5", 2019) assert ( url - == "https://geo.public.data.uu.nl/vault-futurestreams/research-futurestreams%5B1633685642%5D/original/" - + "waterTemp/rcp8p5/noresm/waterTemp_weekAvg_output_noresm_rcp8p5_2006-01-07_to_2019-12-30.nc" + == "https://geo.public.data.uu.nl/vault-futurestreams/research-futurestreams%5B1633685642%5D/original/" # noqa:W503 + + "waterTemp/rcp8p5/noresm/waterTemp_weekAvg_output_noresm_rcp8p5_2006-01-07_to_2019-12-30.nc" # noqa:W503 ) @@ -37,7 +36,7 @@ def test_water_temp_above_mocked(): [dataset[key].rename({"lat": "latitude", "lon": "longitude"}) for key in dataset], ) ) - source = TestSource(dataset) + source = TestSource(dataset, [gcm]) target = TestTarget() # cut down the transform model = WaterTemperatureAboveIndicator( diff --git a/tests/test_wind_onboarding.py b/tests/test_wind_onboarding.py index d10ddaf..439f5de 100644 --- a/tests/test_wind_onboarding.py +++ b/tests/test_wind_onboarding.py @@ -2,12 +2,11 @@ import pytest # type: ignore import zarr # type: ignore -from affine import Affine from hazard.onboard.storm_wind import BatchItem, STORMIndicator # type: ignore from hazard.sources.osc_zarr import OscZarr -from .conftest import test_output_dir +from .conftest import test_output_dir # noqa:F401 used, its a fixture @pytest.mark.skip(reason="on-boarding script")