Skip to content

Commit

Permalink
Merge pull request #215 from stac-utils/local_extensions
Browse files Browse the repository at this point in the history 
allow local schemas in stac extensions
  • Loading branch information
@jonhealy1
jonhealy1 authored Sep 6, 2022
2 parents c53a8aa + 1756c6a commit d640e4c
Show file tree
Hide file tree
Showing 17 changed files with 1,399 additions and 60 deletions.
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ repos:
- id: isort
args: ["--profile", "black"]
- repo: https://github.com/psf/black
rev: 20.8b1
rev: 22.3.0
hooks:
- id: black
language_version: python3.8
7 changes: 6 additions & 1 deletion CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,13 @@ The format is (loosely) based on [Keep a Changelog](http://keepachangelog.com/)

## Unreleased

### Changed
### Added

- Added ability to check local schemas in item extensions https://github.com/stac-utils/stac-validator/pull/215
- Added an example on validating a dictionary https://github.com/stac-utils/stac-validator/pull/215

### Changed

- Changed 'ValidationError' error type to 'JSONSchemaValidationError' https://github.com/stac-utils/stac-validator/pull/213

## [v3.1.0] - 2022-04-28
Expand Down
9 changes: 9 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,15 @@ print(stac.message)
]
```
**Dictionary**
```python
from stac_validator import stac_validator

stac = stac_validator.StacValidate()
stac.validate_dict(dictionary)
print(stac.message)
```
---
# Testing
Expand Down
3 changes: 3 additions & 0 deletions stac_validator/utilities.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import functools
import json
import os
from urllib.parse import urlparse
from urllib.request import urlopen

Expand All @@ -14,6 +15,8 @@
"1.0.0",
]

_pathlib = os.path


def is_url(url: str):
try:
Expand Down
26 changes: 17 additions & 9 deletions stac_validator/validate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
fetch_and_parse_file,
fetch_and_parse_schema,
get_stac_type,
is_valid_url,
link_request,
set_schema_addr,
)
Expand Down Expand Up @@ -97,16 +98,16 @@ def links_validator(self) -> dict:
# get root_url for checking relative links
root_url = ""
for link in self.stac_content["links"]:
if link["rel"] == "self" and link["href"][0:4] == "http":
if link["rel"] == "self" and is_valid_url(link["href"]):
root_url = (
link["href"].split("/")[0] + "//" + link["href"].split("/")[2]
)
elif link["rel"] == "alternate" and link["href"][0:4] == "http":
elif link["rel"] == "alternate" and is_valid_url(link["href"]):
root_url = (
link["href"].split("/")[0] + "//" + link["href"].split("/")[2]
)
for link in self.stac_content["links"]:
if link["href"][0:4] != "http":
if not is_valid_url(link["href"]):
link["href"] = root_url + link["href"][1:]
link_request(link, initial_message)

Expand All @@ -125,7 +126,7 @@ def extensions_validator(self, stac_type: str) -> dict:
self.stac_content["stac_extensions"][index] = "projection"
schemas = self.stac_content["stac_extensions"]
for extension in schemas:
if "http" not in extension:
if not (is_valid_url(extension) or extension.endswith(".json")):
# where are the extensions for 1.0.0-beta.2 on cdn.staclint.com?
if self.version == "1.0.0-beta.2":
self.stac_content["stac_version"] = "1.0.0-beta.1"
Expand Down Expand Up @@ -153,16 +154,23 @@ def extensions_validator(self, stac_type: str) -> dict:
return message

def custom_validator(self):
# in case the path to custom json schema is local
# it may contain relative references
schema = fetch_and_parse_schema(self.custom)
if os.path.exists(self.custom):
# if schema is hosted online
if is_valid_url(self.custom):
schema = fetch_and_parse_schema(self.custom)
jsonschema.validate(self.stac_content, schema)
# in case the path to a json schema is local
elif os.path.exists(self.custom):
schema = fetch_and_parse_schema(self.custom)
custom_abspath = os.path.abspath(self.custom)
custom_dir = os.path.dirname(custom_abspath).replace("\\", "/")
custom_uri = f"file:///{custom_dir}/"
resolver = RefResolver(custom_uri, self.custom)
jsonschema.validate(self.stac_content, schema, resolver=resolver)
# deal with a relative path in the schema
else:
file_directory = os.path.dirname(os.path.abspath(self.stac_file))
self.custom = os.path.join(file_directory, self.custom)
self.custom = os.path.abspath(os.path.realpath(self.custom))
schema = fetch_and_parse_schema(self.custom)
jsonschema.validate(self.stac_content, schema)

Expand Down Expand Up @@ -216,7 +224,7 @@ def recursive_validator(self, stac_type: str) -> bool:
for link in self.stac_content["links"]:
if link["rel"] == "child" or link["rel"] == "item":
address = link["href"]
if "http" not in address:
if not is_valid_url(address):
x = str(base_url).split("/")
x.pop(-1)
st = x[0]
Expand Down
23 changes: 7 additions & 16 deletions tests/test_assets.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,13 @@ def test_assets_v090():


def test_assets_v100():
stac_file = "tests/test_data/v100/core-item.json"
stac_file = "tests/test_data/v100/simple-item.json"
stac = stac_validator.StacValidate(stac_file, assets=True)
stac.run()
assert stac.message == [
{
"version": "1.0.0",
"path": "tests/test_data/v100/core-item.json",
"path": "tests/test_data/v100/simple-item.json",
"schema": [
"https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json"
],
Expand All @@ -66,23 +66,14 @@ def test_assets_v100():
"validation_method": "default",
"assets_validated": {
"format_valid": [
"https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2_analytic.tif",
"https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.jpg",
"https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.tif",
"https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2_analytic_udm.tif",
"http://remotedata.io/catalog/20201211_223832_CS2/extended-metadata.json",
"http://cool-sat.com/catalog/20201211_223832_CS2/20201211_223832_CS2.EPH",
"https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2_test.tif",
"https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2_test.jpg",
],
"format_invalid": [],
"request_valid": [
"https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2_analytic.tif",
"https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.jpg",
"https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.tif",
"https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2_analytic_udm.tif",
"http://remotedata.io/catalog/20201211_223832_CS2/extended-metadata.json",
],
"request_valid": [],
"request_invalid": [
"http://cool-sat.com/catalog/20201211_223832_CS2/20201211_223832_CS2.EPH"
"https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2_test.tif",
"https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2_test.jpg",
],
},
}
Expand Down
97 changes: 92 additions & 5 deletions tests/test_custom.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,8 @@ def test_custom_item_remote_schema_v1rc2():
]


def test_custom_eo_error_v1rc2():
schema = "https://stac-extensions.github.io/eo/v1.0.0/schema.json"
def test_custom_proj_error_v1rc2():
schema = "https://stac-extensions.github.io/projection/v1.0.0/schema.json"
stac_file = (
"tests/test_data/1rc2/extensions-collection/./proj-example/proj-example.json"
)
Expand All @@ -111,11 +111,98 @@ def test_custom_eo_error_v1rc2():
{
"version": "1.0.0-rc.2",
"path": "tests/test_data/1rc2/extensions-collection/./proj-example/proj-example.json",
"schema": ["https://stac-extensions.github.io/eo/v1.0.0/schema.json"],
"schema": [
"https://stac-extensions.github.io/projection/v1.0.0/schema.json"
],
"valid_stac": False,
"asset_type": "ITEM",
"validation_method": "custom",
"valid_stac": False,
"error_type": "JSONSchemaValidationError",
"error_message": "'panchromatic' is not one of ['coastal', 'blue', 'green', 'red', 'rededge', 'yellow', 'pan', 'nir', 'nir08', 'nir09', 'cirrus', 'swir16', 'swir22', 'lwir', 'lwir11', 'lwir12']. Error is in assets -> B8 -> eo:bands -> 0 -> common_name ",
"error_message": "'A' is not of type 'number'. Error is in properties -> proj:centroid -> lat ",
}
]


def test_custom_item_v100_relative_schema():
schema = "../schema/v1.0.0/projection.json"
stac_file = "tests/test_data/v100/extended-item-no-extensions.json"
stac = stac_validator.StacValidate(stac_file, custom=schema)
stac.run()
assert stac.message == [
{
"version": "1.0.0",
"path": "tests/test_data/v100/extended-item-no-extensions.json",
"schema": ["../schema/v1.0.0/projection.json"],
"valid_stac": True,
"asset_type": "ITEM",
"validation_method": "custom",
}
]


def test_custom_item_v100_relative_schema_embedded():
schema = "../../schema/v1.0.0/projection.json"
stac_file = "tests/test_data/v100/embedded/extended-item-no-extensions.json"
stac = stac_validator.StacValidate(stac_file, custom=schema)
stac.run()
assert stac.message == [
{
"version": "1.0.0",
"path": "tests/test_data/v100/embedded/extended-item-no-extensions.json",
"schema": ["../../schema/v1.0.0/projection.json"],
"valid_stac": True,
"asset_type": "ITEM",
"validation_method": "custom",
}
]


def test_custom_item_v100_relative_schema_embedded_same_folder():
schema = "./projection.json"
stac_file = "tests/test_data/v100/embedded/extended-item-no-extensions.json"
stac = stac_validator.StacValidate(stac_file, custom=schema)
stac.run()
assert stac.message == [
{
"version": "1.0.0",
"path": "tests/test_data/v100/embedded/extended-item-no-extensions.json",
"schema": ["./projection.json"],
"valid_stac": True,
"asset_type": "ITEM",
"validation_method": "custom",
}
]


def test_custom_item_v100_relative_schema_embedded_same_folder_2():
schema = "projection.json"
stac_file = "tests/test_data/v100/embedded/extended-item-no-extensions.json"
stac = stac_validator.StacValidate(stac_file, custom=schema)
stac.run()
assert stac.message == [
{
"version": "1.0.0",
"path": "tests/test_data/v100/embedded/extended-item-no-extensions.json",
"schema": ["projection.json"],
"valid_stac": True,
"asset_type": "ITEM",
"validation_method": "custom",
}
]


def test_custom_item_v100_local_schema():
schema = "tests/test_data/schema/v1.0.0/projection.json"
stac_file = "tests/test_data/v100/extended-item-no-extensions.json"
stac = stac_validator.StacValidate(stac_file, custom=schema)
stac.run()
assert stac.message == [
{
"version": "1.0.0",
"path": "tests/test_data/v100/extended-item-no-extensions.json",
"schema": ["tests/test_data/schema/v1.0.0/projection.json"],
"valid_stac": True,
"asset_type": "ITEM",
"validation_method": "custom",
}
]
22 changes: 2 additions & 20 deletions tests/test_data/1rc2/extensions-collection/proj-example/proj-example.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@
3951000
],
"proj:centroid": {
"lat": 34.595302781575604,
"lat": "A",
"lon": -101.34448382627504
},
"proj:shape": [
Expand Down Expand Up @@ -246,21 +246,6 @@
"center_wavelength": 0.59,
"full_width_half_max": 0.18
}
],
"proj:shape": [
16781,
16621
],
"proj:transform": [
15,
0,
224992.5,
0,
-15,
6790207.5,
0,
0,
1
]
}
},
Expand All @@ -270,9 +255,6 @@
152.52758,
60.63437
],
"stac_extensions": [
"https://stac-extensions.github.io/eo/v1.0.0/schema.json",
"https://stac-extensions.github.io/projection/v1.0.0/schema.json"
],
"stac_extensions": [],
"collection": "landsat-8-l1"
}
Loading

0 comments on commit d640e4c

Please sign in to comment.