diff --git a/src/backend/app/projects/image_processing.py b/src/backend/app/projects/image_processing.py index 03cc7cf0..4e5baea7 100644 --- a/src/backend/app/projects/image_processing.py +++ b/src/backend/app/projects/image_processing.py @@ -172,6 +172,10 @@ async def _process_images( # Check and add the GCP file to the images list if it exists if get_file_from_bucket(bucket_name, gcp_list_file, gcp_file_path): images_list.append(gcp_file_path) + else: + log.info( + f"GCP file not available for project ID {self.project_id}." + ) for task_id in self.task_ids: self.download_images_from_s3(bucket_name, temp_dir, task_id) @@ -376,9 +380,9 @@ async def process_assets_from_odm( raise log.info(f"Successfully downloaded ZIP to {assets_path}") - s3_path = f"dtm-data/projects/{dtm_project_id}/{dtm_task_id if dtm_task_id else ''}/assets.zip".strip( - "/" - ) + # Construct the S3 path dynamically to avoid empty segments + task_segment = f"{dtm_task_id}/" if dtm_task_id else "" + s3_path = f"dtm-data/projects/{dtm_project_id}/{task_segment}assets.zip" log.info(f"Uploading {assets_path} to S3 path: {s3_path}") add_file_to_bucket(settings.S3_BUCKET_NAME, assets_path, s3_path) @@ -393,17 +397,14 @@ async def process_assets_from_odm( raise FileNotFoundError("Orthophoto file is missing") reproject_to_web_mercator(orthophoto_path, orthophoto_path) - s3_ortho_path = f"dtm-data/projects/{dtm_project_id}/{dtm_task_id if dtm_task_id else ''}/orthophoto/odm_orthophoto.tif".strip( - "/" - ) - + s3_ortho_path = f"dtm-data/projects/{dtm_project_id}/{task_segment}orthophoto/odm_orthophoto.tif" log.info(f"Uploading reprojected orthophoto to S3 path: {s3_ortho_path}") add_file_to_bucket(settings.S3_BUCKET_NAME, orthophoto_path, s3_ortho_path) images_json_path = os.path.join(output_file_path, "images.json") if os.path.exists(images_json_path): - s3_images_json_path = f"dtm-data/projects/{dtm_project_id}/{dtm_task_id if dtm_task_id else ''}/images.json".strip( - "/" + s3_images_json_path = ( + f"dtm-data/projects/{dtm_project_id}/{task_segment}images.json" ) log.info(f"Uploading images.json to S3 path: {s3_images_json_path}") add_file_to_bucket( diff --git a/src/backend/app/s3.py b/src/backend/app/s3.py index afdc3778..dbcec789 100644 --- a/src/backend/app/s3.py +++ b/src/backend/app/s3.py @@ -5,6 +5,7 @@ from typing import Any from datetime import timedelta from urllib.parse import urljoin +from minio.error import S3Error def s3_client(): @@ -104,9 +105,15 @@ def get_file_from_bucket(bucket_name: str, s3_path: str, file_path: str): # Ensure s3_path starts with a forward slash # if not s3_path.startswith("/"): # s3_path = f"/{s3_path}" - - client = s3_client() - client.fget_object(bucket_name, s3_path, file_path) + try: + client = s3_client() + client.fget_object(bucket_name, s3_path, file_path) + except S3Error as e: + if e.code == "NoSuchKey": + log.warning(f"File not found in bucket: {s3_path}") + else: + log.error(f"Error occurred while downloading file: {e}") + return False def get_obj_from_bucket(bucket_name: str, s3_path: str) -> BytesIO: diff --git a/src/backend/app/tasks/task_routes.py b/src/backend/app/tasks/task_routes.py index 2138d2b6..6264828f 100644 --- a/src/backend/app/tasks/task_routes.py +++ b/src/backend/app/tasks/task_routes.py @@ -167,157 +167,162 @@ async def update_task_table( db, project_id, task_id, "assets_url", s3_path_url ) - if task["total_image_uploaded"] is None: - await cur.execute( - """ - SELECT state - FROM task_events - WHERE task_id = %s AND state = 'IMAGE_UPLOADED' - ORDER BY created_at DESC - """, - (task_id,), - ) - task_event = await cur.fetchone() - if task_event: - # update the count of the task to image uploaded. - toatl_image_count = project_logic.get_project_info_from_s3( - project_id, task_id - ).image_count - - await project_logic.update_task_field( - db, - project_id, - task_id, - "total_image_uploaded", - toatl_image_count, - ) - - if task["flight_time_minutes"] and task["flight_distance_km"] is None: - import geojson - from drone_flightplan import ( - waypoints, - add_elevation_from_dem, - calculate_parameters, - create_placemarks, - ) - from app.s3 import get_file_from_bucket - from geojson import Feature, FeatureCollection, Polygon - from app.models.enums import FlightMode - from app.utils import calculate_flight_time_from_placemarks - - # Fetch the task outline + if task["total_image_uploaded"] is None: await cur.execute( """ - SELECT jsonb_build_object( - 'type', 'Feature', - 'geometry', ST_AsGeoJSON(tasks.outline)::jsonb, - 'properties', jsonb_build_object( - 'id', tasks.id, - 'bbox', jsonb_build_array( - ST_XMin(ST_Envelope(tasks.outline)), - ST_YMin(ST_Envelope(tasks.outline)), - ST_XMax(ST_Envelope(tasks.outline)), - ST_YMax(ST_Envelope(tasks.outline)) - ) - ), - 'id', tasks.id - ) AS outline - FROM tasks - WHERE id = %s; - """, - (task["id"],), - ) - polygon = await cur.fetchone() - polygon = polygon["outline"] - forward_overlap = ( - project["front_overlap"] if project["front_overlap"] else 70 - ) - side_overlap = ( - project["side_overlap"] if project["side_overlap"] else 70 - ) - generate_3d = False # TODO: For 3d imageries drone_flightplan package needs to be updated. - - gsd = project["gsd_cm_px"] - altitude = project["altitude_from_ground"] - - parameters = calculate_parameters( - forward_overlap, - side_overlap, - altitude, - gsd, - 2, + SELECT state + FROM task_events + WHERE task_id = %s AND state = 'IMAGE_UPLOADED' + ORDER BY created_at DESC + """, + (task_id,), ) + task_event = await cur.fetchone() + if task_event: + # update the count of the task to image uploaded. + toatl_image_count = project_logic.get_project_info_from_s3( + project_id, task_id + ).image_count - # Wrap polygon into GeoJSON Feature - coordinates = polygon["geometry"]["coordinates"] - if polygon["geometry"]["type"] == "Polygon": - coordinates = polygon["geometry"]["coordinates"] - feature = Feature(geometry=Polygon(coordinates), properties={}) - feature_collection = FeatureCollection([feature]) - - # Common parameters for create_waypoint - waypoint_params = { - "project_area": feature_collection, - "agl": altitude, - "gsd": gsd, - "forward_overlap": forward_overlap, - "side_overlap": side_overlap, - "rotation_angle": 0, - "generate_3d": generate_3d, - } - waypoint_params["mode"] = FlightMode.waypoints - if project["is_terrain_follow"]: - dem_path = f"/tmp/{uuid.uuid4()}/dem.tif" - - # Terrain follow uses waypoints mode, waylines are generated later - points = waypoints.create_waypoint(**waypoint_params) - - try: - get_file_from_bucket( - settings.S3_BUCKET_NAME, - f"dtm-data/projects/{project.id}/dem.tif", - dem_path, - ) - # TODO: Do this with inmemory data - outfile_with_elevation = ( - "/tmp/output_file_with_elevation.geojson" - ) - add_elevation_from_dem( - dem_path, points, outfile_with_elevation - ) + await project_logic.update_task_field( + db, + project_id, + task_id, + "total_image_uploaded", + toatl_image_count, + ) - inpointsfile = open(outfile_with_elevation, "r") - points_with_elevation = inpointsfile.read() + if ( + task["flight_time_minutes"] + and task["flight_distance_km"] is None + ): + import geojson + from drone_flightplan import ( + waypoints, + add_elevation_from_dem, + calculate_parameters, + create_placemarks, + ) + from app.s3 import get_file_from_bucket + from geojson import Feature, FeatureCollection, Polygon + from app.models.enums import FlightMode + from app.utils import calculate_flight_time_from_placemarks + + # Fetch the task outline + await cur.execute( + """ + SELECT jsonb_build_object( + 'type', 'Feature', + 'geometry', ST_AsGeoJSON(tasks.outline)::jsonb, + 'properties', jsonb_build_object( + 'id', tasks.id, + 'bbox', jsonb_build_array( + ST_XMin(ST_Envelope(tasks.outline)), + ST_YMin(ST_Envelope(tasks.outline)), + ST_XMax(ST_Envelope(tasks.outline)), + ST_YMax(ST_Envelope(tasks.outline)) + ) + ), + 'id', tasks.id + ) AS outline + FROM tasks + WHERE id = %s; + """, + (task["id"],), + ) + polygon = await cur.fetchone() + polygon = polygon["outline"] + forward_overlap = ( + project["front_overlap"] if project["front_overlap"] else 70 + ) + side_overlap = ( + project["side_overlap"] if project["side_overlap"] else 70 + ) + generate_3d = False - except Exception: - points_with_elevation = points + gsd = project["gsd_cm_px"] + altitude = project["altitude_from_ground"] - placemarks = create_placemarks( - geojson.loads(points_with_elevation), parameters + parameters = calculate_parameters( + forward_overlap, + side_overlap, + altitude, + gsd, + 2, ) - else: - points = waypoints.create_waypoint(**waypoint_params) - placemarks = create_placemarks( - geojson.loads(points), parameters - ) + # Wrap polygon into GeoJSON Feature + coordinates = polygon["geometry"]["coordinates"] + + if polygon["geometry"]["type"] == "Polygon": + coordinates = polygon["geometry"]["coordinates"] + + feature = Feature(geometry=Polygon(coordinates), properties={}) + feature_collection = FeatureCollection([feature]) + + # Common parameters for create_waypoint + waypoint_params = { + "project_area": feature_collection, + "agl": altitude, + "gsd": gsd, + "forward_overlap": forward_overlap, + "side_overlap": side_overlap, + "rotation_angle": 0, + "generate_3d": generate_3d, + } + waypoint_params["mode"] = FlightMode.waypoints + if project["is_terrain_follow"]: + dem_path = f"/tmp/{uuid.uuid4()}/dem.tif" + + # Terrain follow uses waypoints mode, waylines are generated later + points = waypoints.create_waypoint(**waypoint_params) + + try: + get_file_from_bucket( + settings.S3_BUCKET_NAME, + f"dtm-data/projects/{project.id}/dem.tif", + dem_path, + ) + # TODO: Do this with inmemory data + outfile_with_elevation = ( + "/tmp/output_file_with_elevation.geojson" + ) + add_elevation_from_dem( + dem_path, points, outfile_with_elevation + ) + + inpointsfile = open(outfile_with_elevation, "r") + points_with_elevation = inpointsfile.read() + + except Exception: + points_with_elevation = points + + placemarks = create_placemarks( + geojson.loads(points_with_elevation), parameters + ) - flight_time_minutes = calculate_flight_time_from_placemarks( - placemarks - ).get("total_flight_time") - flight_distance_km = calculate_flight_time_from_placemarks( - placemarks - ).get("flight_distance_km") + else: + points = waypoints.create_waypoint(**waypoint_params) + placemarks = create_placemarks( + geojson.loads(points), parameters + ) - # Update the total_area_sqkm in the tasks table - await cur.execute( - """ - UPDATE tasks - SET flight_time_minutes = %s, - flight_distance_km = %s - WHERE id = %s - """, - (flight_time_minutes, flight_distance_km, task["id"]), - ) + flight_time_minutes = calculate_flight_time_from_placemarks( + placemarks + ).get("total_flight_time") + flight_distance_km = calculate_flight_time_from_placemarks( + placemarks + ).get("flight_distance_km") + + # Update the total_area_sqkm in the tasks table + await cur.execute( + """ + UPDATE tasks + SET flight_time_minutes = %s, + flight_distance_km = %s + WHERE id = %s + """, + (flight_time_minutes, flight_distance_km, task["id"]), + ) return {"message": "Task table updated successfully."}