Skip to content

Commit

Permalink
test: w problematic feed
Browse files Browse the repository at this point in the history
  • Loading branch information
cka-y committed Oct 7, 2023
1 parent 9aab47a commit ce73e28
Showing 1 changed file with 5 additions and 2 deletions.
7 changes: 5 additions & 2 deletions infra/batch/datasets/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -208,14 +208,14 @@ def process_dataset(cloud_event: CloudEvent):
"""
stable_id = "UNKNOWN"
error_return_message = f'ERROR - Unsuccessful processing of dataset with stable id {stable_id}.'
bucket_name = os.getenv("BUCKET_NAME")
try:
data = base64.b64decode(cloud_event.data["message"]["data"]).decode()
json_payload = json.loads(data)
producer_url, stable_id, dataset_hash, dataset_id = json_payload["producer_url"], json_payload["stable_id"], \
json_payload["dataset_hash"], json_payload["dataset_id"]

print(f"[{stable_id} INFO] JSON Payload:", json_payload)
bucket_name = os.getenv("BUCKET_NAME")

if dataset_id is None:
print(f"[{stable_id} INTERNAL ERROR] Couldn't find latest dataset related to feed_id.\n")
Expand All @@ -225,10 +225,13 @@ def process_dataset(cloud_event: CloudEvent):
print(f'[{stable_id} INFO] Process completed. No database update required.')
except Exception as e:
print(f'[{stable_id} ERROR] Error while uploading dataset\n {e} \n {traceback.format_exc()}')
handle_error(bucket_name, e, "", stable_id)
return error_return_message

# Allow raised exception to trigger the retry process until a connection is available
engine = get_db_engine()
validate_dataset_version(engine.connect(), json_payload, bucket_name, sha256_file_hash, hosted_url)

return 'Done!'


Expand All @@ -246,7 +249,7 @@ def batch_datasets(request):
# Retrieve feeds
engine = get_db_engine()
sql_statement = "select stable_id, producer_url, gtfsfeed.id from feed join gtfsfeed on gtfsfeed.id=feed.id where " \
"status='active' and authentication_type='0'"
"status='active' and authentication_type='0' and stable_id='mdb-1061'"
results = engine.execute(text(sql_statement)).all()
print(f"Retrieved {len(results)} active feeds.")

Expand Down

0 comments on commit ce73e28

Please sign in to comment.