Skip to content

Commit

Permalink
fix: update all image processing without gcp files
Browse files Browse the repository at this point in the history 
issues solve on task table update
  • Loading branch information
@Pradip-p
Pradip-p committed Dec 31, 2024
1 parent e6b2759 commit ec648e5
Show file tree
Hide file tree
Showing 3 changed files with 166 additions and 153 deletions.
19 changes: 10 additions & 9 deletions src/backend/app/projects/image_processing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,10 @@ async def _process_images(
# Check and add the GCP file to the images list if it exists
if get_file_from_bucket(bucket_name, gcp_list_file, gcp_file_path):
images_list.append(gcp_file_path)
else:
log.info(
f"GCP file not available for project ID {self.project_id}."
)

for task_id in self.task_ids:
self.download_images_from_s3(bucket_name, temp_dir, task_id)
Expand Down Expand Up @@ -376,9 +380,9 @@ async def process_assets_from_odm(
raise
log.info(f"Successfully downloaded ZIP to {assets_path}")

s3_path = f"dtm-data/projects/{dtm_project_id}/{dtm_task_id if dtm_task_id else ''}/assets.zip".strip(
"/"
)
# Construct the S3 path dynamically to avoid empty segments
task_segment = f"{dtm_task_id}/" if dtm_task_id else ""
s3_path = f"dtm-data/projects/{dtm_project_id}/{task_segment}assets.zip"
log.info(f"Uploading {assets_path} to S3 path: {s3_path}")
add_file_to_bucket(settings.S3_BUCKET_NAME, assets_path, s3_path)

Expand All @@ -393,17 +397,14 @@ async def process_assets_from_odm(
raise FileNotFoundError("Orthophoto file is missing")

reproject_to_web_mercator(orthophoto_path, orthophoto_path)
s3_ortho_path = f"dtm-data/projects/{dtm_project_id}/{dtm_task_id if dtm_task_id else ''}/orthophoto/odm_orthophoto.tif".strip(
"/"
)

s3_ortho_path = f"dtm-data/projects/{dtm_project_id}/{task_segment}orthophoto/odm_orthophoto.tif"
log.info(f"Uploading reprojected orthophoto to S3 path: {s3_ortho_path}")
add_file_to_bucket(settings.S3_BUCKET_NAME, orthophoto_path, s3_ortho_path)

images_json_path = os.path.join(output_file_path, "images.json")
if os.path.exists(images_json_path):
s3_images_json_path = f"dtm-data/projects/{dtm_project_id}/{dtm_task_id if dtm_task_id else ''}/images.json".strip(
"/"
s3_images_json_path = (
f"dtm-data/projects/{dtm_project_id}/{task_segment}images.json"
)
log.info(f"Uploading images.json to S3 path: {s3_images_json_path}")
add_file_to_bucket(
Expand Down
13 changes: 10 additions & 3 deletions src/backend/app/s3.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from typing import Any
from datetime import timedelta
from urllib.parse import urljoin
from minio.error import S3Error


def s3_client():
Expand Down Expand Up @@ -104,9 +105,15 @@ def get_file_from_bucket(bucket_name: str, s3_path: str, file_path: str):
# Ensure s3_path starts with a forward slash
# if not s3_path.startswith("/"):
# s3_path = f"/{s3_path}"

client = s3_client()
client.fget_object(bucket_name, s3_path, file_path)
try:
client = s3_client()
client.fget_object(bucket_name, s3_path, file_path)
except S3Error as e:
if e.code == "NoSuchKey":
log.warning(f"File not found in bucket: {s3_path}")
else:
log.error(f"Error occurred while downloading file: {e}")
return False


def get_obj_from_bucket(bucket_name: str, s3_path: str) -> BytesIO:
Expand Down
287 changes: 146 additions & 141 deletions src/backend/app/tasks/task_routes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,157 +167,162 @@ async def update_task_table(
db, project_id, task_id, "assets_url", s3_path_url
)

if task["total_image_uploaded"] is None:
await cur.execute(
"""
SELECT state
FROM task_events
WHERE task_id = %s AND state = 'IMAGE_UPLOADED'
ORDER BY created_at DESC
""",
(task_id,),
)
task_event = await cur.fetchone()
if task_event:
# update the count of the task to image uploaded.
toatl_image_count = project_logic.get_project_info_from_s3(
project_id, task_id
).image_count

await project_logic.update_task_field(
db,
project_id,
task_id,
"total_image_uploaded",
toatl_image_count,
)

if task["flight_time_minutes"] and task["flight_distance_km"] is None:
import geojson
from drone_flightplan import (
waypoints,
add_elevation_from_dem,
calculate_parameters,
create_placemarks,
)
from app.s3 import get_file_from_bucket
from geojson import Feature, FeatureCollection, Polygon
from app.models.enums import FlightMode
from app.utils import calculate_flight_time_from_placemarks

# Fetch the task outline
if task["total_image_uploaded"] is None:
await cur.execute(
"""
SELECT jsonb_build_object(
'type', 'Feature',
'geometry', ST_AsGeoJSON(tasks.outline)::jsonb,
'properties', jsonb_build_object(
'id', tasks.id,
'bbox', jsonb_build_array(
ST_XMin(ST_Envelope(tasks.outline)),
ST_YMin(ST_Envelope(tasks.outline)),
ST_XMax(ST_Envelope(tasks.outline)),
ST_YMax(ST_Envelope(tasks.outline))
)
),
'id', tasks.id
) AS outline
FROM tasks
WHERE id = %s;
""",
(task["id"],),
)
polygon = await cur.fetchone()
polygon = polygon["outline"]
forward_overlap = (
project["front_overlap"] if project["front_overlap"] else 70
)
side_overlap = (
project["side_overlap"] if project["side_overlap"] else 70
)
generate_3d = False # TODO: For 3d imageries drone_flightplan package needs to be updated.

gsd = project["gsd_cm_px"]
altitude = project["altitude_from_ground"]

parameters = calculate_parameters(
forward_overlap,
side_overlap,
altitude,
gsd,
2,
SELECT state
FROM task_events
WHERE task_id = %s AND state = 'IMAGE_UPLOADED'
ORDER BY created_at DESC
""",
(task_id,),
)
task_event = await cur.fetchone()
if task_event:
# update the count of the task to image uploaded.
toatl_image_count = project_logic.get_project_info_from_s3(
project_id, task_id
).image_count

# Wrap polygon into GeoJSON Feature
coordinates = polygon["geometry"]["coordinates"]
if polygon["geometry"]["type"] == "Polygon":
coordinates = polygon["geometry"]["coordinates"]
feature = Feature(geometry=Polygon(coordinates), properties={})
feature_collection = FeatureCollection([feature])

# Common parameters for create_waypoint
waypoint_params = {
"project_area": feature_collection,
"agl": altitude,
"gsd": gsd,
"forward_overlap": forward_overlap,
"side_overlap": side_overlap,
"rotation_angle": 0,
"generate_3d": generate_3d,
}
waypoint_params["mode"] = FlightMode.waypoints
if project["is_terrain_follow"]:
dem_path = f"/tmp/{uuid.uuid4()}/dem.tif"

# Terrain follow uses waypoints mode, waylines are generated later
points = waypoints.create_waypoint(**waypoint_params)

try:
get_file_from_bucket(
settings.S3_BUCKET_NAME,
f"dtm-data/projects/{project.id}/dem.tif",
dem_path,
)
# TODO: Do this with inmemory data
outfile_with_elevation = (
"/tmp/output_file_with_elevation.geojson"
)
add_elevation_from_dem(
dem_path, points, outfile_with_elevation
)
await project_logic.update_task_field(
db,
project_id,
task_id,
"total_image_uploaded",
toatl_image_count,
)

inpointsfile = open(outfile_with_elevation, "r")
points_with_elevation = inpointsfile.read()
if (
task["flight_time_minutes"]
and task["flight_distance_km"] is None
):
import geojson
from drone_flightplan import (
waypoints,
add_elevation_from_dem,
calculate_parameters,
create_placemarks,
)
from app.s3 import get_file_from_bucket
from geojson import Feature, FeatureCollection, Polygon
from app.models.enums import FlightMode
from app.utils import calculate_flight_time_from_placemarks

# Fetch the task outline
await cur.execute(
"""
SELECT jsonb_build_object(
'type', 'Feature',
'geometry', ST_AsGeoJSON(tasks.outline)::jsonb,
'properties', jsonb_build_object(
'id', tasks.id,
'bbox', jsonb_build_array(
ST_XMin(ST_Envelope(tasks.outline)),
ST_YMin(ST_Envelope(tasks.outline)),
ST_XMax(ST_Envelope(tasks.outline)),
ST_YMax(ST_Envelope(tasks.outline))
)
),
'id', tasks.id
) AS outline
FROM tasks
WHERE id = %s;
""",
(task["id"],),
)
polygon = await cur.fetchone()
polygon = polygon["outline"]
forward_overlap = (
project["front_overlap"] if project["front_overlap"] else 70
)
side_overlap = (
project["side_overlap"] if project["side_overlap"] else 70
)
generate_3d = False

except Exception:
points_with_elevation = points
gsd = project["gsd_cm_px"]
altitude = project["altitude_from_ground"]

placemarks = create_placemarks(
geojson.loads(points_with_elevation), parameters
parameters = calculate_parameters(
forward_overlap,
side_overlap,
altitude,
gsd,
2,
)

else:
points = waypoints.create_waypoint(**waypoint_params)
placemarks = create_placemarks(
geojson.loads(points), parameters
)
# Wrap polygon into GeoJSON Feature
coordinates = polygon["geometry"]["coordinates"]

if polygon["geometry"]["type"] == "Polygon":
coordinates = polygon["geometry"]["coordinates"]

feature = Feature(geometry=Polygon(coordinates), properties={})
feature_collection = FeatureCollection([feature])

# Common parameters for create_waypoint
waypoint_params = {
"project_area": feature_collection,
"agl": altitude,
"gsd": gsd,
"forward_overlap": forward_overlap,
"side_overlap": side_overlap,
"rotation_angle": 0,
"generate_3d": generate_3d,
}
waypoint_params["mode"] = FlightMode.waypoints
if project["is_terrain_follow"]:
dem_path = f"/tmp/{uuid.uuid4()}/dem.tif"

# Terrain follow uses waypoints mode, waylines are generated later
points = waypoints.create_waypoint(**waypoint_params)

try:
get_file_from_bucket(
settings.S3_BUCKET_NAME,
f"dtm-data/projects/{project.id}/dem.tif",
dem_path,
)
# TODO: Do this with inmemory data
outfile_with_elevation = (
"/tmp/output_file_with_elevation.geojson"
)
add_elevation_from_dem(
dem_path, points, outfile_with_elevation
)

inpointsfile = open(outfile_with_elevation, "r")
points_with_elevation = inpointsfile.read()

except Exception:
points_with_elevation = points

placemarks = create_placemarks(
geojson.loads(points_with_elevation), parameters
)

flight_time_minutes = calculate_flight_time_from_placemarks(
placemarks
).get("total_flight_time")
flight_distance_km = calculate_flight_time_from_placemarks(
placemarks
).get("flight_distance_km")
else:
points = waypoints.create_waypoint(**waypoint_params)
placemarks = create_placemarks(
geojson.loads(points), parameters
)

# Update the total_area_sqkm in the tasks table
await cur.execute(
"""
UPDATE tasks
SET flight_time_minutes = %s,
flight_distance_km = %s
WHERE id = %s
""",
(flight_time_minutes, flight_distance_km, task["id"]),
)
flight_time_minutes = calculate_flight_time_from_placemarks(
placemarks
).get("total_flight_time")
flight_distance_km = calculate_flight_time_from_placemarks(
placemarks
).get("flight_distance_km")

# Update the total_area_sqkm in the tasks table
await cur.execute(
"""
UPDATE tasks
SET flight_time_minutes = %s,
flight_distance_km = %s
WHERE id = %s
""",
(flight_time_minutes, flight_distance_km, task["id"]),
)

return {"message": "Task table updated successfully."}

0 comments on commit ec648e5

Please sign in to comment.