introducing delete items upon de-labeling

FAIRiCUBE · Oct 6, 2023 · 007b680 · 007b680
1 parent 1ae7175
commit 007b680
Show file tree

Hide file tree

Showing 67 changed files with 64 additions and 3,042 deletions.
diff --git a/stac/stac-generator/stac-generator.py b/stac/stac-generator/stac-generator.py
@@ -284,7 +284,7 @@ def insert_value(field, value_list, listed_fields, doc):
 index_catalog = pystac.Catalog.from_file('./stac_dist/catalog.json')
 for index, issue in enumerate(issues):
     labels = issue["node"]["labels"]["edges"]
-    requested = {'node': {'name': 'data-request'}}
+    requested = {'node': {'name': 'test-approved'}}
     if requested in labels:
 
         title = issue["node"]["title"].split("]: ")[-1].replace(" ", "_")

diff --git a/stac/stac-generator/update_items.py b/stac/stac-generator/update_items.py
@@ -1,14 +1,38 @@
 import requests
 import os
+import pystac
 from requests.auth import HTTPBasicAuth
 
 username = os.environ["USERNAME"]
 password = os.environ["PASSWORD"]
 
 catalog_url = "https://fairicube.github.io/data-requests/catalog.json"
 post_url = "https://stacapi-write.eoxhub.fairicube.eu/collections/index/items"
-# post_url = "http://0.0.0.0:8082/collections/index/items"
+
+
+def delete_item(id):
+    response = requests.delete(url=f"{post_url}/{id}",
+                               auth=HTTPBasicAuth(username, password))
+    return response.status_code
+
+
 index_catalog = requests.get(catalog_url).json()
+catalog = pystac.Catalog.from_dict(index_catalog)
+catalog_items = catalog.get_items()
+
+catalog_items_list = []
+
+for item in catalog_items:
+    catalog_items_list.append(item.id)
+
+api_items = requests.get(url=f"{post_url}?limit=1000",
+                         auth=HTTPBasicAuth(username, password)).json()
+
+# check if the already ingested items are in the catalog, otherwise delete them
+for item in api_items["features"]:
+    if item["id"] not in catalog_items_list:
+        delete_item(item["id"])
+
 for link in index_catalog["links"]:
     if link["rel"] == "item":
         json_body = requests.get(link["href"]).json()

diff --git a/...A38_and_the_United_Kingdom/Administrative_boundaries_of_EEA38_and_the_United_Kingdom.json b/...A38_and_the_United_Kingdom/Administrative_boundaries_of_EEA38_and_the_United_Kingdom.json
diff --git a/stac_dist/COPERNICUS_30/COPERNICUS_30.json b/stac_dist/COPERNICUS_30/COPERNICUS_30.json
@@ -221,7 +221,7 @@
     90
   ],
   "stac_extensions": [
-    "https://stac-extensions.github.io/raster/v1.0.0/schema.json",
+    "https://stac-extensions.github.io/raster/v1.1.0/schema.json",
     "https://stac-extensions.github.io/datacube/v2.0.0/schema.json"
   ]
 }
diff --git a/stac_dist/CORINE_LAND_COVER/CORINE_LAND_COVER.json b/stac_dist/CORINE_LAND_COVER/CORINE_LAND_COVER.json
@@ -191,7 +191,7 @@
     71
   ],
   "stac_extensions": [
-    "https://stac-extensions.github.io/raster/v1.0.0/schema.json",
+    "https://stac-extensions.github.io/raster/v1.1.0/schema.json",
     "https://stac-extensions.github.io/datacube/v2.0.0/schema.json"
   ]
 }
diff --git a/stac_dist/CORINE_LAND_COVER_ACCOUNTING_LAYERS/CORINE_LAND_COVER_ACCOUNTING_LAYERS.json b/stac_dist/CORINE_LAND_COVER_ACCOUNTING_LAYERS/CORINE_LAND_COVER_ACCOUNTING_LAYERS.json
@@ -186,7 +186,7 @@
     72
   ],
   "stac_extensions": [
-    "https://stac-extensions.github.io/raster/v1.0.0/schema.json",
+    "https://stac-extensions.github.io/raster/v1.1.0/schema.json",
     "https://stac-extensions.github.io/datacube/v2.0.0/schema.json"
   ]
 }
diff --git a/stac_dist/City___Urban_Audit_2021_10m_raster/City___Urban_Audit_2021_10m_raster.json b/stac_dist/City___Urban_Audit_2021_10m_raster/City___Urban_Audit_2021_10m_raster.json
diff --git a/..._Data_-_Near-surface_specific_humidity/Climate_Data_-_Near-surface_specific_humidity.json b/..._Data_-_Near-surface_specific_humidity/Climate_Data_-_Near-surface_specific_humidity.json