From 48722b831725c56e4dfda18430c53751063922d5 Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Thu, 26 Dec 2024 13:53:34 +0545 Subject: [PATCH 01/36] feat: update the gcp files for all project images --- src/backend/app/gcp/gcp_routes.py | 12 ++++++------ src/backend/app/projects/image_processing.py | 7 +++++++ src/backend/app/projects/project_routes.py | 13 ++++++++++++- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/backend/app/gcp/gcp_routes.py b/src/backend/app/gcp/gcp_routes.py index 31799e50..bc68cdba 100644 --- a/src/backend/app/gcp/gcp_routes.py +++ b/src/backend/app/gcp/gcp_routes.py @@ -1,5 +1,6 @@ import uuid from app.config import settings +from app.projects import project_schemas from fastapi import APIRouter, Depends from app.waypoints import waypoint_schemas from app.gcp import gcp_crud @@ -21,15 +22,15 @@ async def find_images( project_id: uuid.UUID, task_id: uuid.UUID, + db: Annotated[Connection, Depends(database.get_db)], point: waypoint_schemas.PointField = None, ) -> List[str]: """Find images that contain a specified point.""" fov_degree = 82.1 # For DJI Mini 4 Pro - altitude = 100 # TODO: Get this from db - + result = await project_schemas.DbProject.one(db, project_id) return await gcp_crud.find_images_in_a_task_for_point( - project_id, task_id, point, fov_degree, altitude + project_id, task_id, point, fov_degree, result.altitude ) @@ -42,11 +43,10 @@ async def find_images_for_a_project( """Find images that contain a specified point in a project.""" fov_degree = 82.1 # For DJI Mini 4 Pro - altitude = 100 # TODO: Get this from db - + result = await project_schemas.DbProject.one(db, project_id) # Get all task IDs for the project from database task_id_list = await list_task_id_for_project(db, project_id) return await gcp_crud.find_images_in_a_project_for_point( - project_id, task_id_list, point, fov_degree, altitude + project_id, task_id_list, point, fov_degree, result.altitude ) diff --git a/src/backend/app/projects/image_processing.py b/src/backend/app/projects/image_processing.py index ec0a2de7..e7757486 100644 --- a/src/backend/app/projects/image_processing.py +++ b/src/backend/app/projects/image_processing.py @@ -165,6 +165,13 @@ async def _process_images( self.download_images_from_s3(bucket_name, temp_dir, self.task_id) images_list = self.list_images(temp_dir) else: + gcp_list_file = f"dtm-data/projects/{self.project_id}/gcp/gcp_list.txt" + gcp_file_path = os.path.join(temp_dir, "gcp_list.txt") + + # Check and add the GCP file to the images list if it exists + if get_file_from_bucket(bucket_name, gcp_list_file, gcp_file_path): + images_list.append(gcp_file_path) + for task_id in self.task_ids: self.download_images_from_s3(bucket_name, temp_dir, task_id) images_list.extend(self.list_images(temp_dir)) diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index cc9e43c1..cfe23cc8 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -467,12 +467,23 @@ async def process_all_imagery( user_data: Annotated[AuthUser, Depends(login_required)], background_tasks: BackgroundTasks, db: Annotated[Connection, Depends(database.get_db)], + gcp_file: UploadFile = File(None), + ): """ API endpoint to process all tasks associated with a project. """ user_id = user_data.id - + if gcp_file: + gcp_file_path = f"/tmp/{uuid.uuid4()}" + with open(gcp_file_path, "wb") as f: + f.write(await gcp_file.read()) + + s3_path = ( + f"dtm-data/projects/{project_id}/gcp/gcp_list.txt" + ) + add_file_to_bucket( settings.S3_BUCKET_NAME, gcp_file_path, s3_path) + tasks = await project_logic.get_all_tasks_for_project(project.id, db) background_tasks.add_task( project_logic.process_all_drone_images, project_id, tasks, user_id, db From 3391e63840a80c0bf98b6ad059c4a509a24658ff Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Thu, 26 Dec 2024 14:13:15 +0545 Subject: [PATCH 02/36] feat: update the new fields in tasks tables --- src/backend/app/db/db_models.py | 4 ++ .../app/migrations/versions/d202ea539f6d_.py | 42 +++++++++++++++++++ src/backend/app/projects/image_processing.py | 2 +- src/backend/app/projects/project_logic.py | 30 +++++++++++-- src/backend/app/projects/project_routes.py | 11 ++--- 5 files changed, 77 insertions(+), 12 deletions(-) create mode 100644 src/backend/app/migrations/versions/d202ea539f6d_.py diff --git a/src/backend/app/db/db_models.py b/src/backend/app/db/db_models.py index 8b4a2ac7..8d8f159e 100644 --- a/src/backend/app/db/db_models.py +++ b/src/backend/app/db/db_models.py @@ -78,6 +78,10 @@ class DbTask(Base): take_off_point = cast( WKBElement, Column(Geometry("POINT", srid=4326), nullable=True) ) + total_area_sqkm = cast(float, Column(Float, nullable=True)) + flight_time_minutes = cast(int, Column(Float, nullable=True)) + flight_distance_km = cast(float, Column(Float, nullable=True)) + total_image_uploaded = cast(int, Column(SmallInteger, nullable=True)) class DbProject(Base): diff --git a/src/backend/app/migrations/versions/d202ea539f6d_.py b/src/backend/app/migrations/versions/d202ea539f6d_.py new file mode 100644 index 00000000..f6107002 --- /dev/null +++ b/src/backend/app/migrations/versions/d202ea539f6d_.py @@ -0,0 +1,42 @@ +""" + +Revision ID: d202ea539f6d +Revises: e23c05f21542 +Create Date: 2024-12-26 08:11:00.011691 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'd202ea539f6d' +down_revision: Union[str, None] = 'e23c05f21542' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('task_events', 'state', + existing_type=postgresql.ENUM('REQUEST_FOR_MAPPING', 'UNLOCKED_TO_MAP', 'LOCKED_FOR_MAPPING', 'UNLOCKED_TO_VALIDATE', 'LOCKED_FOR_VALIDATION', 'UNLOCKED_DONE', 'UNFLYABLE_TASK', 'IMAGE_UPLOADED', 'IMAGE_PROCESSING_FAILED', 'IMAGE_PROCESSING_STARTED', 'IMAGE_PROCESSING_FINISHED', name='state'), + nullable=False) + op.add_column('tasks', sa.Column('total_area_sqkm', sa.Float(), nullable=True)) + op.add_column('tasks', sa.Column('flight_time_minutes', sa.Float(), nullable=True)) + op.add_column('tasks', sa.Column('flight_distance_km', sa.Float(), nullable=True)) + op.add_column('tasks', sa.Column('total_image_uploaded', sa.SmallInteger(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('tasks', 'total_image_uploaded') + op.drop_column('tasks', 'flight_distance_km') + op.drop_column('tasks', 'flight_time_minutes') + op.drop_column('tasks', 'total_area_sqkm') + op.alter_column('task_events', 'state', + existing_type=postgresql.ENUM('REQUEST_FOR_MAPPING', 'UNLOCKED_TO_MAP', 'LOCKED_FOR_MAPPING', 'UNLOCKED_TO_VALIDATE', 'LOCKED_FOR_VALIDATION', 'UNLOCKED_DONE', 'UNFLYABLE_TASK', 'IMAGE_UPLOADED', 'IMAGE_PROCESSING_FAILED', 'IMAGE_PROCESSING_STARTED', 'IMAGE_PROCESSING_FINISHED', name='state'), + nullable=True) + # ### end Alembic commands ### diff --git a/src/backend/app/projects/image_processing.py b/src/backend/app/projects/image_processing.py index e7757486..d9a4ead2 100644 --- a/src/backend/app/projects/image_processing.py +++ b/src/backend/app/projects/image_processing.py @@ -171,7 +171,7 @@ async def _process_images( # Check and add the GCP file to the images list if it exists if get_file_from_bucket(bucket_name, gcp_list_file, gcp_file_path): images_list.append(gcp_file_path) - + for task_id in self.task_ids: self.download_images_from_s3(bucket_name, temp_dir, task_id) images_list.extend(self.list_images(temp_dir)) diff --git a/src/backend/app/projects/project_logic.py b/src/backend/app/projects/project_logic.py index af1769ad..184a032e 100644 --- a/src/backend/app/projects/project_logic.py +++ b/src/backend/app/projects/project_logic.py @@ -150,10 +150,11 @@ async def create_tasks_from_geojson( async with db.cursor() as cur: await cur.execute( """ - INSERT INTO tasks (id, project_id, outline, project_task_index) - VALUES (%(id)s, %(project_id)s, %(outline)s, %(project_task_index)s) - RETURNING id; - """, + INSERT INTO tasks (id, project_id, outline, project_task_index, total_area_sqkm) + VALUES (%(id)s, %(project_id)s, %(outline)s, %(project_task_index)s, + ST_Area(ST_Transform(ST_SetSRID(outline, 4326), 3857)) / 1000000) + RETURNING id; + """, { "id": task_id, "project_id": project_id, @@ -163,7 +164,28 @@ async def create_tasks_from_geojson( "project_task_index": index + 1, }, ) + + # async with db.cursor() as cur: + # await cur.execute( + # """ + # INSERT INTO tasks (id, project_id, outline, project_task_index) + # VALUES (%(id)s, %(project_id)s, %(outline)s, %(project_task_index)s) + # RETURNING id; + # ST_Area(ST_Transform(t.outline, 3857)) / 1000000 AS task_area + + # """, + # { + # "id": task_id, + # "project_id": project_id, + # "outline": wkblib.dumps( + # shape(polygon["geometry"]), hex=True + # ), + # "project_task_index": index + 1, + # }, + # ) + result = await cur.fetchone() + if result: log.debug( "Created database task | " diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index cfe23cc8..ce4c06f5 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -468,7 +468,6 @@ async def process_all_imagery( background_tasks: BackgroundTasks, db: Annotated[Connection, Depends(database.get_db)], gcp_file: UploadFile = File(None), - ): """ API endpoint to process all tasks associated with a project. @@ -478,12 +477,10 @@ async def process_all_imagery( gcp_file_path = f"/tmp/{uuid.uuid4()}" with open(gcp_file_path, "wb") as f: f.write(await gcp_file.read()) - - s3_path = ( - f"dtm-data/projects/{project_id}/gcp/gcp_list.txt" - ) - add_file_to_bucket( settings.S3_BUCKET_NAME, gcp_file_path, s3_path) - + + s3_path = f"dtm-data/projects/{project_id}/gcp/gcp_list.txt" + add_file_to_bucket(settings.S3_BUCKET_NAME, gcp_file_path, s3_path) + tasks = await project_logic.get_all_tasks_for_project(project.id, db) background_tasks.add_task( project_logic.process_all_drone_images, project_id, tasks, user_id, db From 7774eefcf76d0d1b0a7b59fa01077e7af8a235e0 Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Thu, 26 Dec 2024 15:13:44 +0545 Subject: [PATCH 03/36] feat: update task areas from db to instead of postgis --- .../app/migrations/versions/d202ea539f6d_.py | 67 ++++++++++++++----- src/backend/app/projects/project_logic.py | 55 ++++++++------- src/backend/app/projects/project_schemas.py | 9 ++- src/backend/app/tasks/task_schemas.py | 8 +-- .../Dashboard/TaskLogs/TaskLogsTable.tsx | 2 +- .../DescriptionBox/index.tsx | 4 +- .../Tasks/TableSection/index.tsx | 4 +- 7 files changed, 91 insertions(+), 58 deletions(-) diff --git a/src/backend/app/migrations/versions/d202ea539f6d_.py b/src/backend/app/migrations/versions/d202ea539f6d_.py index f6107002..9830f62e 100644 --- a/src/backend/app/migrations/versions/d202ea539f6d_.py +++ b/src/backend/app/migrations/versions/d202ea539f6d_.py @@ -5,6 +5,7 @@ Create Date: 2024-12-26 08:11:00.011691 """ + from typing import Sequence, Union from alembic import op @@ -12,31 +13,65 @@ from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision: str = 'd202ea539f6d' -down_revision: Union[str, None] = 'e23c05f21542' +revision: str = "d202ea539f6d" +down_revision: Union[str, None] = "e23c05f21542" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('task_events', 'state', - existing_type=postgresql.ENUM('REQUEST_FOR_MAPPING', 'UNLOCKED_TO_MAP', 'LOCKED_FOR_MAPPING', 'UNLOCKED_TO_VALIDATE', 'LOCKED_FOR_VALIDATION', 'UNLOCKED_DONE', 'UNFLYABLE_TASK', 'IMAGE_UPLOADED', 'IMAGE_PROCESSING_FAILED', 'IMAGE_PROCESSING_STARTED', 'IMAGE_PROCESSING_FINISHED', name='state'), - nullable=False) - op.add_column('tasks', sa.Column('total_area_sqkm', sa.Float(), nullable=True)) - op.add_column('tasks', sa.Column('flight_time_minutes', sa.Float(), nullable=True)) - op.add_column('tasks', sa.Column('flight_distance_km', sa.Float(), nullable=True)) - op.add_column('tasks', sa.Column('total_image_uploaded', sa.SmallInteger(), nullable=True)) + op.alter_column( + "task_events", + "state", + existing_type=postgresql.ENUM( + "REQUEST_FOR_MAPPING", + "UNLOCKED_TO_MAP", + "LOCKED_FOR_MAPPING", + "UNLOCKED_TO_VALIDATE", + "LOCKED_FOR_VALIDATION", + "UNLOCKED_DONE", + "UNFLYABLE_TASK", + "IMAGE_UPLOADED", + "IMAGE_PROCESSING_FAILED", + "IMAGE_PROCESSING_STARTED", + "IMAGE_PROCESSING_FINISHED", + name="state", + ), + nullable=False, + ) + op.add_column("tasks", sa.Column("total_area_sqkm", sa.Float(), nullable=True)) + op.add_column("tasks", sa.Column("flight_time_minutes", sa.Float(), nullable=True)) + op.add_column("tasks", sa.Column("flight_distance_km", sa.Float(), nullable=True)) + op.add_column( + "tasks", sa.Column("total_image_uploaded", sa.SmallInteger(), nullable=True) + ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('tasks', 'total_image_uploaded') - op.drop_column('tasks', 'flight_distance_km') - op.drop_column('tasks', 'flight_time_minutes') - op.drop_column('tasks', 'total_area_sqkm') - op.alter_column('task_events', 'state', - existing_type=postgresql.ENUM('REQUEST_FOR_MAPPING', 'UNLOCKED_TO_MAP', 'LOCKED_FOR_MAPPING', 'UNLOCKED_TO_VALIDATE', 'LOCKED_FOR_VALIDATION', 'UNLOCKED_DONE', 'UNFLYABLE_TASK', 'IMAGE_UPLOADED', 'IMAGE_PROCESSING_FAILED', 'IMAGE_PROCESSING_STARTED', 'IMAGE_PROCESSING_FINISHED', name='state'), - nullable=True) + op.drop_column("tasks", "total_image_uploaded") + op.drop_column("tasks", "flight_distance_km") + op.drop_column("tasks", "flight_time_minutes") + op.drop_column("tasks", "total_area_sqkm") + op.alter_column( + "task_events", + "state", + existing_type=postgresql.ENUM( + "REQUEST_FOR_MAPPING", + "UNLOCKED_TO_MAP", + "LOCKED_FOR_MAPPING", + "UNLOCKED_TO_VALIDATE", + "LOCKED_FOR_VALIDATION", + "UNLOCKED_DONE", + "UNFLYABLE_TASK", + "IMAGE_UPLOADED", + "IMAGE_PROCESSING_FAILED", + "IMAGE_PROCESSING_STARTED", + "IMAGE_PROCESSING_FINISHED", + name="state", + ), + nullable=True, + ) # ### end Alembic commands ### diff --git a/src/backend/app/projects/project_logic.py b/src/backend/app/projects/project_logic.py index 184a032e..473e4efb 100644 --- a/src/backend/app/projects/project_logic.py +++ b/src/backend/app/projects/project_logic.py @@ -2,7 +2,6 @@ import uuid from loguru import logger as log from fastapi import HTTPException, UploadFile -from pyproj import Transformer from app.tasks.task_splitter import split_by_square from fastapi.concurrency import run_in_threadpool from psycopg import Connection @@ -21,6 +20,8 @@ from app.projects import project_schemas from minio import S3Error from psycopg.rows import dict_row +from shapely.ops import transform +import pyproj async def get_centroids(db: Connection): @@ -127,12 +128,20 @@ async def create_tasks_from_geojson( if isinstance(boundaries, str): boundaries = json.loads(boundaries) - # Update the boundary polyon on the database. if boundaries["type"] == "Feature": polygons = [boundaries] else: polygons = boundaries["features"] + log.debug(f"Processing {len(polygons)} task geometries") + + # Set up the projection transform for EPSG:3857 (Web Mercator) + proj_wgs84 = pyproj.CRS("EPSG:4326") + proj_mercator = pyproj.CRS("EPSG:3857") + project_transformer = pyproj.Transformer.from_crs( + proj_wgs84, proj_mercator, always_xy=True + ) + for index, polygon in enumerate(polygons): try: if not polygon["geometry"]: @@ -141,18 +150,25 @@ async def create_tasks_from_geojson( if polygon["geometry"]["type"] == "MultiPolygon": log.debug("Converting MultiPolygon to Polygon") polygon["geometry"]["type"] = "Polygon" - polygon["geometry"]["coordinates"] = polygon["geometry"][ "coordinates" ][0] + geom = shape(polygon["geometry"]) + + # Transform the geometry to EPSG:3857 and calculate the area in square meters + transformed_geom = transform(project_transformer.transform, geom) + area_sq_m = transformed_geom.area # Area in square meters + + # Convert area to square kilometers + total_area_sqkm = area_sq_m / 1_000_000 + task_id = str(uuid.uuid4()) async with db.cursor() as cur: await cur.execute( """ INSERT INTO tasks (id, project_id, outline, project_task_index, total_area_sqkm) - VALUES (%(id)s, %(project_id)s, %(outline)s, %(project_task_index)s, - ST_Area(ST_Transform(ST_SetSRID(outline, 4326), 3857)) / 1000000) + VALUES (%(id)s, %(project_id)s, %(outline)s, %(project_task_index)s, %(total_area_sqkm)s) RETURNING id; """, { @@ -162,30 +178,11 @@ async def create_tasks_from_geojson( shape(polygon["geometry"]), hex=True ), "project_task_index": index + 1, + "total_area_sqkm": total_area_sqkm, }, ) - - # async with db.cursor() as cur: - # await cur.execute( - # """ - # INSERT INTO tasks (id, project_id, outline, project_task_index) - # VALUES (%(id)s, %(project_id)s, %(outline)s, %(project_task_index)s) - # RETURNING id; - # ST_Area(ST_Transform(t.outline, 3857)) / 1000000 AS task_area - - # """, - # { - # "id": task_id, - # "project_id": project_id, - # "outline": wkblib.dumps( - # shape(polygon["geometry"]), hex=True - # ), - # "project_task_index": index + 1, - # }, - # ) - result = await cur.fetchone() - + if result: log.debug( "Created database task | " @@ -342,8 +339,10 @@ async def check_regulator_project(db: Connection, project_id: str, email: str): def generate_square_geojson(center_lat, center_lon, side_length_meters): - transformer = Transformer.from_crs("EPSG:4326", "EPSG:3857", always_xy=True) - transformer_back = Transformer.from_crs("EPSG:3857", "EPSG:4326", always_xy=True) + transformer = pyproj.Transformer.from_crs("EPSG:4326", "EPSG:3857", always_xy=True) + transformer_back = pyproj.Transformer.from_crs( + "EPSG:3857", "EPSG:4326", always_xy=True + ) center_x, center_y = transformer.transform(center_lon, center_lat) half_side = side_length_meters / 2 diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 962b2f4c..09be3fae 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -173,10 +173,10 @@ class TaskOut(BaseModel): outline: Optional[Polygon | Feature | FeatureCollection] = None state: Optional[str] = None user_id: Optional[str] = None - task_area: Optional[float] = None name: Optional[str] = None image_count: Optional[int] = None assets_url: Optional[str] = None + total_area_sqkm: Optional[float] = None class DbProject(BaseModel): @@ -210,7 +210,6 @@ class DbProject(BaseModel): is_terrain_follow: bool = False image_url: Optional[str] = None created_at: datetime - author_id: str async def one(db: Connection, project_id: uuid.UUID): """Get a single project & all associated tasks by ID.""" @@ -291,6 +290,7 @@ async def one(db: Connection, project_id: uuid.UUID): t.id, t.project_task_index, t.project_id, + t.total_area_sqkm, ST_AsGeoJSON(t.outline)::jsonb -> 'coordinates' AS coordinates, ST_AsGeoJSON(t.outline)::jsonb -> 'type' AS type, ST_XMin(ST_Envelope(t.outline)) AS xmin, @@ -299,8 +299,7 @@ async def one(db: Connection, project_id: uuid.UUID): ST_YMax(ST_Envelope(t.outline)) AS ymax, tsc.state AS state, tsc.user_id, - u.name, - ST_Area(ST_Transform(t.outline, 3857)) / 1000000 AS task_area + u.name FROM tasks t LEFT JOIN @@ -316,8 +315,8 @@ async def one(db: Connection, project_id: uuid.UUID): state, user_id, name, - task_area, project_id, + total_area_sqkm, jsonb_build_object( 'type', 'Feature', 'geometry', jsonb_build_object( diff --git a/src/backend/app/tasks/task_schemas.py b/src/backend/app/tasks/task_schemas.py index 31715c2e..9f66265e 100644 --- a/src/backend/app/tasks/task_schemas.py +++ b/src/backend/app/tasks/task_schemas.py @@ -165,7 +165,7 @@ async def all(db: Connection, project_id: uuid.UUID): class UserTasksStatsOut(BaseModel): task_id: uuid.UUID - task_area: float + total_area_sqkm: Optional[float] = None created_at: datetime state: str project_id: uuid.UUID @@ -207,7 +207,7 @@ async def get_tasks_by_user( tasks.project_task_index AS project_task_index, task_events.project_id AS project_id, projects.name AS project_name, - ST_Area(ST_Transform(tasks.outline, 3857)) / 1000000 AS task_area, + tasks.total_area_sqkm, task_events.created_at, task_events.updated_at, task_events.state, @@ -263,7 +263,7 @@ async def get_tasks_by_user( class TaskDetailsOut(BaseModel): - task_area: float + total_area_sqkm: float outline: Outline created_at: datetime updated_at: Optional[datetime] = None @@ -300,7 +300,7 @@ async def get_task_details(db: Connection, task_id: uuid.UUID): await cur.execute( """ SELECT - ST_Area(ST_Transform(tasks.outline, 3857)) / 1000000 AS task_area, + tasks.total_area_sqkm, -- Construct the outline as a GeoJSON Feature jsonb_build_object( diff --git a/src/frontend/src/components/Dashboard/TaskLogs/TaskLogsTable.tsx b/src/frontend/src/components/Dashboard/TaskLogs/TaskLogsTable.tsx index a46a31f2..77e5f007 100644 --- a/src/frontend/src/components/Dashboard/TaskLogs/TaskLogsTable.tsx +++ b/src/frontend/src/components/Dashboard/TaskLogs/TaskLogsTable.tsx @@ -42,7 +42,7 @@ const TaskLogsTable = ({ data: taskList }: ITaskLogsTableProps) => { {task?.project_name} - {Number(task?.task_area)?.toFixed(3)} + {Number(task?.total_area_sqkm)?.toFixed(3)} {/* - */} diff --git a/src/frontend/src/components/DroneOperatorTask/DescriptionSection/DescriptionBox/index.tsx b/src/frontend/src/components/DroneOperatorTask/DescriptionSection/DescriptionBox/index.tsx index fcc90b63..5cd5e9e2 100644 --- a/src/frontend/src/components/DroneOperatorTask/DescriptionSection/DescriptionBox/index.tsx +++ b/src/frontend/src/components/DroneOperatorTask/DescriptionSection/DescriptionBox/index.tsx @@ -135,8 +135,8 @@ const DescriptionBox = () => { }, { name: 'Total task area', - value: taskData?.task_area - ? `${Number(taskData?.task_area)?.toFixed(3)} km²` + value: taskData?.total_area_sqkm + ? `${Number(taskData?.total_area_sqkm)?.toFixed(3)} km²` : null, }, { diff --git a/src/frontend/src/components/IndividualProject/Tasks/TableSection/index.tsx b/src/frontend/src/components/IndividualProject/Tasks/TableSection/index.tsx index 340ab791..22c3f20e 100644 --- a/src/frontend/src/components/IndividualProject/Tasks/TableSection/index.tsx +++ b/src/frontend/src/components/IndividualProject/Tasks/TableSection/index.tsx @@ -10,7 +10,7 @@ const tasksDataColumns = [ { header: 'Task Area in km²', accessorKey: 'task_area', - }, + } ]; interface ITableSectionProps { @@ -34,7 +34,7 @@ export default function TableSection({ { id: `Task# ${curr?.project_task_index}`, flight_time: curr?.flight_time || '-', - task_area: Number(curr?.task_area)?.toFixed(3), + task_area: Number(curr?.total_area_sqkm)?.toFixed(3), task_id: curr?.id, // status: curr?.state, }, From ba0df7f3b152d84002bc91d61e0167243c4c40ff Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Mon, 30 Dec 2024 13:05:32 +0545 Subject: [PATCH 04/36] feat: update task flight time, flight distance & task areas --- src/backend/app/models/enums.py | 2 +- src/backend/app/projects/project_logic.py | 113 +++++++++++++++++- src/backend/app/projects/project_routes.py | 7 +- src/backend/app/projects/project_schemas.py | 6 + src/backend/app/tasks/task_logic.py | 9 ++ src/backend/app/tasks/task_schemas.py | 11 +- src/backend/app/utils.py | 23 ++-- .../Tasks/TableSection/index.tsx | 10 ++ 8 files changed, 160 insertions(+), 21 deletions(-) diff --git a/src/backend/app/models/enums.py b/src/backend/app/models/enums.py index 76d54c8d..cb6cebad 100644 --- a/src/backend/app/models/enums.py +++ b/src/backend/app/models/enums.py @@ -125,7 +125,7 @@ class DroneType(IntEnum): DJI_MINI_4_PRO = 1 -class UserRole(IntEnum, Enum): +class UserRole(int, Enum): PROJECT_CREATOR = 1 DRONE_PILOT = 2 REGULATOR = 3 diff --git a/src/backend/app/projects/project_logic.py b/src/backend/app/projects/project_logic.py index 473e4efb..a632a17b 100644 --- a/src/backend/app/projects/project_logic.py +++ b/src/backend/app/projects/project_logic.py @@ -22,6 +22,19 @@ from psycopg.rows import dict_row from shapely.ops import transform import pyproj +from geojson import Feature, FeatureCollection, Polygon +from app.s3 import get_file_from_bucket +from app.utils import ( + calculate_flight_time_from_placemarks, +) +import geojson +from drone_flightplan import ( + waypoints, + add_elevation_from_dem, + calculate_parameters, + create_placemarks, +) +from app.models.enums import FlightMode async def get_centroids(db: Connection): @@ -122,6 +135,7 @@ async def create_tasks_from_geojson( db: Connection, project_id: uuid.UUID, boundaries: str, + project: project_schemas.DbProject, ): """Create tasks for a project, from provided task boundaries.""" try: @@ -143,6 +157,77 @@ async def create_tasks_from_geojson( ) for index, polygon in enumerate(polygons): + forward_overlap = project.front_overlap if project.front_overlap else 70 + side_overlap = project.side_overlap if project.side_overlap else 70 + generate_3d = False # TODO: For 3d imageries drone_flightplan package needs to be updated. + + gsd = project.gsd_cm_px + altitude = project.altitude_from_ground + + parameters = calculate_parameters( + forward_overlap, + side_overlap, + altitude, + gsd, + 2, # Image Interval is set to 2 + ) + + # Wrap polygon into GeoJSON Feature + coordinates = polygon["geometry"]["coordinates"] + if polygon["geometry"]["type"] == "Polygon": + coordinates = polygon["geometry"]["coordinates"] + feature = Feature(geometry=Polygon(coordinates), properties={}) + feature_collection = FeatureCollection([feature]) + + # Common parameters for create_waypoint + waypoint_params = { + "project_area": feature_collection, + "agl": altitude, + "gsd": gsd, + "forward_overlap": forward_overlap, + "side_overlap": side_overlap, + "rotation_angle": 0, + "generate_3d": generate_3d, + } + waypoint_params["mode"] = FlightMode.waypoints + if project.is_terrain_follow: + dem_path = f"/tmp/{uuid.uuid4()}/dem.tif" + + # Terrain follow uses waypoints mode, waylines are generated later + points = waypoints.create_waypoint(**waypoint_params) + + try: + get_file_from_bucket( + settings.S3_BUCKET_NAME, + f"dtm-data/projects/{project.id}/dem.tif", + dem_path, + ) + # TODO: Do this with inmemory data + outfile_with_elevation = "/tmp/output_file_with_elevation.geojson" + add_elevation_from_dem(dem_path, points, outfile_with_elevation) + + inpointsfile = open(outfile_with_elevation, "r") + points_with_elevation = inpointsfile.read() + + except Exception: + points_with_elevation = points + + placemarks = create_placemarks( + geojson.loads(points_with_elevation), parameters + ) + + else: + points = waypoints.create_waypoint(**waypoint_params) + placemarks = create_placemarks(geojson.loads(points), parameters) + + flight_time_minutes = calculate_flight_time_from_placemarks(placemarks).get( + "total_flight_time" + ) + flight_distance_km = calculate_flight_time_from_placemarks(placemarks).get( + "flight_distance_km" + ) + print(f"Flight time: {flight_time_minutes} minutes") + print(f"Flight distance: {flight_distance_km} km") try: if not polygon["geometry"]: continue @@ -167,8 +252,8 @@ async def create_tasks_from_geojson( async with db.cursor() as cur: await cur.execute( """ - INSERT INTO tasks (id, project_id, outline, project_task_index, total_area_sqkm) - VALUES (%(id)s, %(project_id)s, %(outline)s, %(project_task_index)s, %(total_area_sqkm)s) + INSERT INTO tasks (id, project_id, outline, project_task_index, total_area_sqkm, flight_time_minutes, flight_distance_km) + VALUES (%(id)s, %(project_id)s, %(outline)s, %(project_task_index)s, %(total_area_sqkm)s, %(flight_time_minutes)s, %(flight_distance_km)s) RETURNING id; """, { @@ -179,6 +264,8 @@ async def create_tasks_from_geojson( ), "project_task_index": index + 1, "total_area_sqkm": total_area_sqkm, + "flight_time_minutes": flight_time_minutes, + "flight_distance_km": flight_distance_km, }, ) result = await cur.fetchone() @@ -383,3 +470,25 @@ async def get_all_tasks_for_project(project_id, db): results = await cur.fetchall() # Convert UUIDs to string return [str(result[0]) for result in results] + + +async def update_total_image_uploaded( + db: Connection, project_id: uuid.UUID, task_id: uuid.UUID, total_image_count: str +): + """ + Update the total_image_uploaded field in the tasks table. + """ + async with db.cursor() as cur: + await cur.execute( + """ + UPDATE tasks + SET total_image_uploaded = %(total_image_uploaded)s + WHERE project_id = %(project_id)s AND id = %(task_id)s; + """, + { + "total_image_uploaded": total_image_count, + "project_id": str(project_id), + "task_id": str(task_id), + }, + ) + return True diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index ce4c06f5..28f284d1 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -257,7 +257,7 @@ async def upload_project_task_boundaries( dict: JSON containing success message, project ID, and number of tasks. """ log.debug("Creating tasks for each polygon in project") - await project_logic.create_tasks_from_geojson(db, project.id, task_featcol) + await project_logic.create_tasks_from_geojson(db, project.id, task_featcol, project) return {"message": "Project Boundary Uploaded", "project_id": f"{project.id}"} @@ -305,9 +305,11 @@ async def preview_split_by_square( @router.post("/generate-presigned-url/", tags=["Image Upload"]) async def generate_presigned_url( + db: Annotated[Connection, Depends(database.get_db)], user: Annotated[AuthUser, Depends(login_required)], data: project_schemas.PresignedUrlRequest, replace_existing: bool = False, + ): """ Generate a pre-signed URL for uploading an image to S3 Bucket. @@ -366,7 +368,7 @@ async def generate_presigned_url( status_code=HTTPStatus.BAD_REQUEST, detail=f"Failed to delete existing image. {e}", ) - + # Generate a new pre-signed URL for the image upload url = client.get_presigned_url( "PUT", @@ -745,7 +747,6 @@ async def get_assets_info( if task_id is None: # Fetch all tasks associated with the project tasks = await project_deps.get_tasks_by_project_id(project.id, db) - results = [] for task in tasks: diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 09be3fae..69239db3 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -177,6 +177,8 @@ class TaskOut(BaseModel): image_count: Optional[int] = None assets_url: Optional[str] = None total_area_sqkm: Optional[float] = None + flight_time_minutes: Optional[float] = None + flight_distance_km: Optional[float] = None class DbProject(BaseModel): @@ -291,6 +293,8 @@ async def one(db: Connection, project_id: uuid.UUID): t.project_task_index, t.project_id, t.total_area_sqkm, + t.flight_time_minutes, + t.flight_distance_km, ST_AsGeoJSON(t.outline)::jsonb -> 'coordinates' AS coordinates, ST_AsGeoJSON(t.outline)::jsonb -> 'type' AS type, ST_XMin(ST_Envelope(t.outline)) AS xmin, @@ -317,6 +321,8 @@ async def one(db: Connection, project_id: uuid.UUID): name, project_id, total_area_sqkm, + flight_distance_km, + flight_time_minutes, jsonb_build_object( 'type', 'Feature', 'geometry', jsonb_build_object( diff --git a/src/backend/app/tasks/task_logic.py b/src/backend/app/tasks/task_logic.py index e7277dec..57215849 100644 --- a/src/backend/app/tasks/task_logic.py +++ b/src/backend/app/tasks/task_logic.py @@ -4,6 +4,7 @@ from app.tasks.task_schemas import NewEvent, TaskStats from app.users import user_schemas from app.utils import render_email_template, send_notification_email +from app.projects import project_logic from psycopg import Connection from app.models.enums import EventType, HTTPStatus, State, UserRole from fastapi import HTTPException, BackgroundTasks @@ -602,6 +603,14 @@ async def handle_event( status_code=403, detail="You cannot upload an image for this task as it is locked by another user.", ) + # update the count of the task to image uploaded. + toatl_image_count = project_logic.get_project_info_from_s3( + project_id, task_id + ).image_count + + await project_logic.update_total_image_uploaded( + db, project_id, task_id, toatl_image_count + ) return await update_task_state( db, diff --git a/src/backend/app/tasks/task_schemas.py b/src/backend/app/tasks/task_schemas.py index 9f66265e..5d10247a 100644 --- a/src/backend/app/tasks/task_schemas.py +++ b/src/backend/app/tasks/task_schemas.py @@ -166,6 +166,9 @@ async def all(db: Connection, project_id: uuid.UUID): class UserTasksStatsOut(BaseModel): task_id: uuid.UUID total_area_sqkm: Optional[float] = None + flight_time_minutes: Optional[float] = None + flight_distance_km: Optional[float] = None + outline: Outline created_at: datetime state: str project_id: uuid.UUID @@ -208,6 +211,8 @@ async def get_tasks_by_user( task_events.project_id AS project_id, projects.name AS project_name, tasks.total_area_sqkm, + tasks.flight_time_minutes, + tasks.flight_distance_km, task_events.created_at, task_events.updated_at, task_events.state, @@ -263,7 +268,9 @@ async def get_tasks_by_user( class TaskDetailsOut(BaseModel): - total_area_sqkm: float + total_area_sqkm: Optional[float] = None + flight_time_minutes: Optional[float] = None + flight_distance_km: Optional[float] = None outline: Outline created_at: datetime updated_at: Optional[datetime] = None @@ -301,6 +308,8 @@ async def get_task_details(db: Connection, task_id: uuid.UUID): """ SELECT tasks.total_area_sqkm, + tasks.flight_time_minutes, + tasks.flight_distance_km, -- Construct the outline as a GeoJSON Feature jsonb_build_object( diff --git a/src/backend/app/utils.py b/src/backend/app/utils.py index 41212919..f3512275 100644 --- a/src/backend/app/utils.py +++ b/src/backend/app/utils.py @@ -558,17 +558,19 @@ async def send_project_approval_email_to_regulator( def calculate_flight_time_from_placemarks(placemarks: Dict) -> Dict: """ - Calculate the total and average flight time based on placemarks and dynamically format the output. + Calculate the total and average flight time and total flight distance based on placemarks. Args: placemarks (Dict): GeoJSON-like data structure with flight plan. Returns: - Dict: Contains formatted total flight time and segment times. + Dict: Contains formatted total flight time, segment times, and total distance. """ total_time = 0 + total_distance = 0 features = placemarks["features"] transformer = Transformer.from_crs("EPSG:4326", "EPSG:3857", always_xy=True) + for i in range(1, len(features)): # Extract current and previous coordinates prev_coords = features[i - 1]["geometry"]["coordinates"][:2] @@ -581,22 +583,15 @@ def calculate_flight_time_from_placemarks(placemarks: Dict) -> Dict: # Calculate distance (meters) and time (seconds) distance = prev_point.distance(curr_point) + total_distance += distance # Accumulate total distance segment_time = distance / speed total_time += segment_time - # Dynamically format the total flight time - hours = int(total_time // 3600) - minutes = int((total_time % 3600) // 60) - seconds = round(total_time % 60, 2) - - if total_time < 60: - formatted_time = f"{seconds} seconds" - elif total_time < 3600: - formatted_time = f"{minutes} minutes {seconds:.2f} seconds" - else: - formatted_time = f"{hours} hours {minutes} minutes {seconds:.2f} seconds" + flight_distance_km = total_distance / 1000 # Convert to kilometers + flight_time_minutes = total_time / 60 # Convert to minutes return { - "total_flight_time": formatted_time, + "total_flight_time": f"{flight_time_minutes:.2f}", "total_flight_time_seconds": round(total_time, 2), + "flight_distance_km": round(flight_distance_km, 2), } diff --git a/src/frontend/src/components/IndividualProject/Tasks/TableSection/index.tsx b/src/frontend/src/components/IndividualProject/Tasks/TableSection/index.tsx index 22c3f20e..156d90a2 100644 --- a/src/frontend/src/components/IndividualProject/Tasks/TableSection/index.tsx +++ b/src/frontend/src/components/IndividualProject/Tasks/TableSection/index.tsx @@ -10,6 +10,14 @@ const tasksDataColumns = [ { header: 'Task Area in km²', accessorKey: 'task_area', + }, + { + header: 'Flight Time in Minutes', + accessorKey: 'flight_time_minutes', + }, + { + header: 'Flight Distance in km', + accessorKey: 'flight_distance_km', } ]; @@ -35,6 +43,8 @@ export default function TableSection({ id: `Task# ${curr?.project_task_index}`, flight_time: curr?.flight_time || '-', task_area: Number(curr?.total_area_sqkm)?.toFixed(3), + flight_time_minutes: Number(curr?.flight_time_minutes)?.toFixed(3), + flight_distance_km: Number(curr?.flight_distance_km)?.toFixed(3), task_id: curr?.id, // status: curr?.state, }, From eb21cf076dd4060fd77bceae8f8547888cc71b3c Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Mon, 30 Dec 2024 13:31:11 +0545 Subject: [PATCH 05/36] fix: import errors in project routes --- src/backend/app/projects/project_routes.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index 28f284d1..d355f644 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -28,7 +28,7 @@ from app.projects import project_schemas, project_deps, project_logic, image_processing from app.db import database from app.models.enums import HTTPStatus, State, FlightMode -from app.s3 import s3_client +from app.s3 import add_file_to_bucket, s3_client from app.config import settings from app.users.user_deps import login_required from app.users.user_schemas import AuthUser @@ -309,7 +309,6 @@ async def generate_presigned_url( user: Annotated[AuthUser, Depends(login_required)], data: project_schemas.PresignedUrlRequest, replace_existing: bool = False, - ): """ Generate a pre-signed URL for uploading an image to S3 Bucket. @@ -368,7 +367,7 @@ async def generate_presigned_url( status_code=HTTPStatus.BAD_REQUEST, detail=f"Failed to delete existing image. {e}", ) - + # Generate a new pre-signed URL for the image upload url = client.get_presigned_url( "PUT", From e457ec2a81697f48b6e59db5fae674596207d91b Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Mon, 30 Dec 2024 15:46:56 +0545 Subject: [PATCH 06/36] fix: waypoints & waylines counts --- src/backend/app/projects/project_logic.py | 116 ++++++++++++++++++++- src/backend/app/projects/project_routes.py | 85 ++------------- 2 files changed, 125 insertions(+), 76 deletions(-) diff --git a/src/backend/app/projects/project_logic.py b/src/backend/app/projects/project_logic.py index a632a17b..0d9d4d42 100644 --- a/src/backend/app/projects/project_logic.py +++ b/src/backend/app/projects/project_logic.py @@ -1,4 +1,6 @@ import json +import os +import shutil import uuid from loguru import logger as log from fastapi import HTTPException, UploadFile @@ -33,6 +35,7 @@ add_elevation_from_dem, calculate_parameters, create_placemarks, + terrain_following_waylines, ) from app.models.enums import FlightMode @@ -226,8 +229,6 @@ async def create_tasks_from_geojson( flight_distance_km = calculate_flight_time_from_placemarks(placemarks).get( "flight_distance_km" ) - print(f"Flight time: {flight_time_minutes} minutes") - print(f"Flight distance: {flight_distance_km} km") try: if not polygon["geometry"]: continue @@ -492,3 +493,114 @@ async def update_total_image_uploaded( }, ) return True + + +async def process_waypoints_and_waylines( + side_overlap: float, + front_overlap: float, + altitude_from_ground: float, + gsd_cm_px: float, + meters: float, + project_geojson: UploadFile, + is_terrain_follow: bool, + dem: UploadFile, +): + """ + Processes and returns counts of waypoints and waylines. + """ + # Validate the input GeoJSON file + file_name = os.path.splitext(project_geojson.filename) + file_ext = file_name[1] + allowed_extensions = [".geojson", ".json"] + if file_ext not in allowed_extensions: + raise HTTPException(status_code=400, detail="Provide a valid .geojson file") + + # Generate square boundary GeoJSON + content = project_geojson.file.read() + boundary = geojson.loads(content) + geometry = shape(boundary["features"][0]["geometry"]) + centroid = geometry.centroid + center_lon = centroid.x + center_lat = centroid.y + square_geojson = generate_square_geojson(center_lat, center_lon, meters) + + # Prepare common parameters for waypoint creation + forward_overlap = front_overlap if front_overlap else 70 + side_overlap = side_overlap if side_overlap else 70 + parameters = calculate_parameters( + forward_overlap, + side_overlap, + altitude_from_ground, + gsd_cm_px, + 2, + ) + waypoint_params = { + "project_area": square_geojson, + "agl": altitude_from_ground, + "gsd": gsd_cm_px, + "forward_overlap": forward_overlap, + "side_overlap": side_overlap, + "rotation_angle": 0, + "generate_3d": False, # TODO: For 3d imageries drone_flightplan package needs to be updated. + "take_off_point": None, + } + count_data = {"waypoints": 0, "waylines": 0} + + if is_terrain_follow and dem: + temp_dir = f"/tmp/{uuid.uuid4()}" + dem_path = os.path.join(temp_dir, "dem.tif") + + try: + os.makedirs(temp_dir, exist_ok=True) + # Read DEM content into memory and write to the file + file_content = await dem.read() + with open(dem_path, "wb") as file: + file.write(file_content) + + # Process waypoints with terrain-follow elevation + waypoint_params["mode"] = FlightMode.waypoints + points = waypoints.create_waypoint(**waypoint_params) + + # Add elevation data to waypoints + outfile_with_elevation = os.path.join( + temp_dir, "output_file_with_elevation.geojson" + ) + add_elevation_from_dem(dem_path, points, outfile_with_elevation) + + # Read the updated waypoints with elevation + with open(outfile_with_elevation, "r") as inpointsfile: + points_with_elevation = inpointsfile.read() + count_data["waypoints"] = len( + json.loads(points_with_elevation)["features"] + ) + + # Generate waylines from waypoints with elevation + wayline_placemarks = create_placemarks( + geojson.loads(points_with_elevation), parameters + ) + + placemarks = terrain_following_waylines.waypoints2waylines( + wayline_placemarks, 5 + ) + count_data["waylines"] = len(placemarks["features"]) + + except Exception as e: + log.error(f"Error processing DEM: {e}") + + finally: + # Cleanup temporary files and directory + if os.path.exists(temp_dir): + shutil.rmtree(temp_dir) + return count_data + + else: + # Generate waypoints and waylines + waypoint_params["mode"] = FlightMode.waypoints + points = waypoints.create_waypoint(**waypoint_params) + count_data["waypoints"] = len(json.loads(points)["features"]) + + waypoint_params["mode"] = FlightMode.waylines + lines = waypoints.create_waypoint(**waypoint_params) + count_data["waylines"] = len(json.loads(lines)["features"]) + + return count_data diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index d355f644..d0505f64 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -1,6 +1,5 @@ import json import os -import shutil import uuid from typing import Annotated, Optional from uuid import UUID @@ -27,7 +26,7 @@ from shapely.ops import unary_union from app.projects import project_schemas, project_deps, project_logic, image_processing from app.db import database -from app.models.enums import HTTPStatus, State, FlightMode +from app.models.enums import HTTPStatus, State from app.s3 import add_file_to_bucket, s3_client from app.config import settings from app.users.user_deps import login_required @@ -41,10 +40,6 @@ from app.users import user_schemas from app.jaxa.upload_dem import upload_dem_file from minio.deleteobjects import DeleteObject -from drone_flightplan import ( - waypoints, - add_elevation_from_dem, -) router = APIRouter( prefix=f"{settings.API_PREFIX}/projects", @@ -654,76 +649,18 @@ async def get_project_waypoints_counts( user_data: AuthUser = Depends(login_required), ): """ - Count waypoints within AOI. + Count waypoints and waylines within AOI. """ - # Validating for .geojson File. - file_name = os.path.splitext(project_geojson.filename) - file_ext = file_name[1] - allowed_extensions = [".geojson", ".json"] - if file_ext not in allowed_extensions: - raise HTTPException(status_code=400, detail="Provide a valid .geojson file") - - # read entire file - content = await project_geojson.read() - boundary = geojson.loads(content) - geometry = shape(boundary["features"][0]["geometry"]) - centroid = geometry.centroid - center_lon = centroid.x - center_lat = centroid.y - square_geojson = project_logic.generate_square_geojson( - center_lat, center_lon, meters - ) - generate_3d = ( - False # TODO: For 3d imageries drone_flightplan package needs to be updated. + return await project_logic.process_waypoints_and_waylines( + side_overlap, + front_overlap, + altitude_from_ground, + gsd_cm_px, + meters, + project_geojson, + is_terrain_follow, + dem, ) - forward_overlap = front_overlap if front_overlap else 70 - side_overlap = side_overlap if side_overlap else 70 - - # Common parameters for create_waypoint - waypoint_params = { - "project_area": square_geojson, - "agl": altitude_from_ground, - "gsd": gsd_cm_px, - "forward_overlap": forward_overlap, - "side_overlap": side_overlap, - "rotation_angle": 0, - "generate_3d": generate_3d, - "take_off_point": None, - } - - waypoint_params["mode"] = FlightMode.waypoints - points = waypoints.create_waypoint(**waypoint_params) - count_data = {"waypoints": 0, "waylines": 0} - - # Handle terrain-following logic if a DEM is provided - if is_terrain_follow and dem: - temp_dir = f"/tmp/{uuid.uuid4()}" - try: - os.makedirs(temp_dir, exist_ok=True) - dem_path = os.path.join(temp_dir, "dem.tif") - outfile_with_elevation = os.path.join( - temp_dir, "output_file_with_elevation.geojson" - ) - - with open(dem_path, "wb") as dem_file: - dem_file.write(await dem.read()) - - add_elevation_from_dem(dem_path, waypoints, outfile_with_elevation) - - except Exception as e: - log.error(f"Error processing DEM: {e}") - - finally: - if os.path.exists(temp_dir): - shutil.rmtree(temp_dir) - - count_data["waypoints"] = len(json.loads(points)["features"]) - else: - waypoint_params["mode"] = FlightMode.waylines - lines = waypoints.create_waypoint(**waypoint_params) - count_data["waypoints"] = len(json.loads(points)["features"]) - count_data["waylines"] = len(json.loads(lines)["features"]) - return count_data @router.get( From 7c7042417dbdebba497b00a230e92187cc91ad5f Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Mon, 30 Dec 2024 16:45:50 +0545 Subject: [PATCH 07/36] fix: only get unique task id based on task events when all image processing.. --- src/backend/app/projects/project_logic.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/backend/app/projects/project_logic.py b/src/backend/app/projects/project_logic.py index 0d9d4d42..4b88e617 100644 --- a/src/backend/app/projects/project_logic.py +++ b/src/backend/app/projects/project_logic.py @@ -459,13 +459,17 @@ def generate_square_geojson(center_lat, center_lon, side_length_meters): async def get_all_tasks_for_project(project_id, db): - "Get all tasks associated with the project ID that are in state IMAGE_UPLOADED." + """ + Get all unique tasks associated with the project ID + that are in state IMAGE_UPLOADED. + """ async with db.cursor() as cur: query = """ - SELECT t.id + SELECT DISTINCT ON (t.id) t.id FROM tasks t JOIN task_events te ON t.id = te.task_id - WHERE t.project_id = %s AND te.state = 'IMAGE_UPLOADED'; + WHERE t.project_id = %s AND te.state = 'IMAGE_UPLOADED' + ORDER BY t.id, te.created_at DESC; """ await cur.execute(query, (project_id,)) results = await cur.fetchall() From bdc8fbaa3840a7dcadc40654ddc6aa76ac6c4c89 Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Mon, 30 Dec 2024 17:06:07 +0545 Subject: [PATCH 08/36] fix: issues resolved in user task out lists in dashboard --- src/backend/app/tasks/task_schemas.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/app/tasks/task_schemas.py b/src/backend/app/tasks/task_schemas.py index 5d10247a..5e2d2111 100644 --- a/src/backend/app/tasks/task_schemas.py +++ b/src/backend/app/tasks/task_schemas.py @@ -168,7 +168,6 @@ class UserTasksStatsOut(BaseModel): total_area_sqkm: Optional[float] = None flight_time_minutes: Optional[float] = None flight_distance_km: Optional[float] = None - outline: Outline created_at: datetime state: str project_id: uuid.UUID From 70b2e60cf8e616746af47dd7169bf0c1d3f99e5c Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Mon, 30 Dec 2024 17:07:58 +0545 Subject: [PATCH 09/36] fixup! fix: issues resolved in user task out lists in dashboard --- src/backend/app/tasks/task_routes.py | 2 +- src/backend/app/tasks/task_schemas.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/app/tasks/task_routes.py b/src/backend/app/tasks/task_routes.py index cc37f152..86cd558a 100644 --- a/src/backend/app/tasks/task_routes.py +++ b/src/backend/app/tasks/task_routes.py @@ -47,7 +47,7 @@ async def list_tasks( user_id = user_data.id role = user_data.role log.info(f"Fetching tasks for user {user_id} with role: {role}") - return await task_schemas.UserTasksStatsOut.get_tasks_by_user( + return await task_schemas.UserTasksOut.get_tasks_by_user( db, user_id, role, skip, limit ) diff --git a/src/backend/app/tasks/task_schemas.py b/src/backend/app/tasks/task_schemas.py index 5e2d2111..286fa7b4 100644 --- a/src/backend/app/tasks/task_schemas.py +++ b/src/backend/app/tasks/task_schemas.py @@ -163,7 +163,7 @@ async def all(db: Connection, project_id: uuid.UUID): return combined_tasks -class UserTasksStatsOut(BaseModel): +class UserTasksOut(BaseModel): task_id: uuid.UUID total_area_sqkm: Optional[float] = None flight_time_minutes: Optional[float] = None @@ -201,7 +201,7 @@ def format_url(url): async def get_tasks_by_user( db: Connection, user_id: str, role: str, skip: int = 0, limit: int = 50 ): - async with db.cursor(row_factory=class_row(UserTasksStatsOut)) as cur: + async with db.cursor(row_factory=class_row(UserTasksOut)) as cur: await cur.execute( """ SELECT DISTINCT ON (tasks.id) From 33ef1da578bea6887156895c331446017df79fd7 Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Mon, 30 Dec 2024 19:16:57 +0545 Subject: [PATCH 10/36] feat: update assests_url in task tables instead of searching in s3 --- src/backend/app/db/db_models.py | 3 + .../app/migrations/versions/b18103ac4ab7_.py | 44 +++++++++++ .../app/migrations/versions/d202ea539f6d_.py | 77 ------------------- src/backend/app/projects/image_processing.py | 8 ++ src/backend/app/projects/project_logic.py | 13 ++-- src/backend/app/projects/project_schemas.py | 16 +++- src/backend/app/s3.py | 23 ++---- src/backend/app/tasks/task_logic.py | 4 +- src/backend/app/tasks/task_schemas.py | 16 +++- 9 files changed, 100 insertions(+), 104 deletions(-) create mode 100644 src/backend/app/migrations/versions/b18103ac4ab7_.py delete mode 100644 src/backend/app/migrations/versions/d202ea539f6d_.py diff --git a/src/backend/app/db/db_models.py b/src/backend/app/db/db_models.py index 8d8f159e..f6c43a21 100644 --- a/src/backend/app/db/db_models.py +++ b/src/backend/app/db/db_models.py @@ -82,6 +82,9 @@ class DbTask(Base): flight_time_minutes = cast(int, Column(Float, nullable=True)) flight_distance_km = cast(float, Column(Float, nullable=True)) total_image_uploaded = cast(int, Column(SmallInteger, nullable=True)) + assets_url = cast( + str, Column(String, nullable=True) + ) # download link for assets of images(orthophoto) class DbProject(Base): diff --git a/src/backend/app/migrations/versions/b18103ac4ab7_.py b/src/backend/app/migrations/versions/b18103ac4ab7_.py new file mode 100644 index 00000000..fc117ba5 --- /dev/null +++ b/src/backend/app/migrations/versions/b18103ac4ab7_.py @@ -0,0 +1,44 @@ +""" + +Revision ID: b18103ac4ab7 +Revises: e23c05f21542 +Create Date: 2024-12-30 11:36:29.762485 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'b18103ac4ab7' +down_revision: Union[str, None] = 'e23c05f21542' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('task_events', 'state', + existing_type=postgresql.ENUM('REQUEST_FOR_MAPPING', 'UNLOCKED_TO_MAP', 'LOCKED_FOR_MAPPING', 'UNLOCKED_TO_VALIDATE', 'LOCKED_FOR_VALIDATION', 'UNLOCKED_DONE', 'UNFLYABLE_TASK', 'IMAGE_UPLOADED', 'IMAGE_PROCESSING_FAILED', 'IMAGE_PROCESSING_STARTED', 'IMAGE_PROCESSING_FINISHED', name='state'), + nullable=False) + op.add_column('tasks', sa.Column('total_area_sqkm', sa.Float(), nullable=True)) + op.add_column('tasks', sa.Column('flight_time_minutes', sa.Float(), nullable=True)) + op.add_column('tasks', sa.Column('flight_distance_km', sa.Float(), nullable=True)) + op.add_column('tasks', sa.Column('total_image_uploaded', sa.SmallInteger(), nullable=True)) + op.add_column('tasks', sa.Column('assets_url', sa.String(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('tasks', 'assets_url') + op.drop_column('tasks', 'total_image_uploaded') + op.drop_column('tasks', 'flight_distance_km') + op.drop_column('tasks', 'flight_time_minutes') + op.drop_column('tasks', 'total_area_sqkm') + op.alter_column('task_events', 'state', + existing_type=postgresql.ENUM('REQUEST_FOR_MAPPING', 'UNLOCKED_TO_MAP', 'LOCKED_FOR_MAPPING', 'UNLOCKED_TO_VALIDATE', 'LOCKED_FOR_VALIDATION', 'UNLOCKED_DONE', 'UNFLYABLE_TASK', 'IMAGE_UPLOADED', 'IMAGE_PROCESSING_FAILED', 'IMAGE_PROCESSING_STARTED', 'IMAGE_PROCESSING_FINISHED', name='state'), + nullable=True) + # ### end Alembic commands ### diff --git a/src/backend/app/migrations/versions/d202ea539f6d_.py b/src/backend/app/migrations/versions/d202ea539f6d_.py deleted file mode 100644 index 9830f62e..00000000 --- a/src/backend/app/migrations/versions/d202ea539f6d_.py +++ /dev/null @@ -1,77 +0,0 @@ -""" - -Revision ID: d202ea539f6d -Revises: e23c05f21542 -Create Date: 2024-12-26 08:11:00.011691 - -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision: str = "d202ea539f6d" -down_revision: Union[str, None] = "e23c05f21542" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.alter_column( - "task_events", - "state", - existing_type=postgresql.ENUM( - "REQUEST_FOR_MAPPING", - "UNLOCKED_TO_MAP", - "LOCKED_FOR_MAPPING", - "UNLOCKED_TO_VALIDATE", - "LOCKED_FOR_VALIDATION", - "UNLOCKED_DONE", - "UNFLYABLE_TASK", - "IMAGE_UPLOADED", - "IMAGE_PROCESSING_FAILED", - "IMAGE_PROCESSING_STARTED", - "IMAGE_PROCESSING_FINISHED", - name="state", - ), - nullable=False, - ) - op.add_column("tasks", sa.Column("total_area_sqkm", sa.Float(), nullable=True)) - op.add_column("tasks", sa.Column("flight_time_minutes", sa.Float(), nullable=True)) - op.add_column("tasks", sa.Column("flight_distance_km", sa.Float(), nullable=True)) - op.add_column( - "tasks", sa.Column("total_image_uploaded", sa.SmallInteger(), nullable=True) - ) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_column("tasks", "total_image_uploaded") - op.drop_column("tasks", "flight_distance_km") - op.drop_column("tasks", "flight_time_minutes") - op.drop_column("tasks", "total_area_sqkm") - op.alter_column( - "task_events", - "state", - existing_type=postgresql.ENUM( - "REQUEST_FOR_MAPPING", - "UNLOCKED_TO_MAP", - "LOCKED_FOR_MAPPING", - "UNLOCKED_TO_VALIDATE", - "LOCKED_FOR_VALIDATION", - "UNLOCKED_DONE", - "UNFLYABLE_TASK", - "IMAGE_UPLOADED", - "IMAGE_PROCESSING_FAILED", - "IMAGE_PROCESSING_STARTED", - "IMAGE_PROCESSING_FINISHED", - name="state", - ), - nullable=True, - ) - # ### end Alembic commands ### diff --git a/src/backend/app/projects/image_processing.py b/src/backend/app/projects/image_processing.py index d9a4ead2..ff5f7ba5 100644 --- a/src/backend/app/projects/image_processing.py +++ b/src/backend/app/projects/image_processing.py @@ -8,6 +8,7 @@ from app.models.enums import State from app.utils import timestamp from app.db import database +from app.projects import project_logic from pyodm import Node from app.s3 import get_file_from_bucket, list_objects_from_bucket, add_file_to_bucket from loguru import logger as log @@ -424,6 +425,13 @@ async def process_assets_from_odm( log.info( f"Task {dtm_task_id} state updated to IMAGE_PROCESSING_FINISHED in the database." ) + s3_path_url = ( + f"dtm-data/projects/{dtm_project_id}/{dtm_task_id}/assets.zip" + ) + # update the task table + await project_logic.update_task_field( + conn, dtm_project_id, dtm_task_id, "assets_url", s3_path_url + ) except Exception as e: log.error(f"Error during processing for project {dtm_project_id}: {e}") diff --git a/src/backend/app/projects/project_logic.py b/src/backend/app/projects/project_logic.py index 4b88e617..f20326ca 100644 --- a/src/backend/app/projects/project_logic.py +++ b/src/backend/app/projects/project_logic.py @@ -1,6 +1,7 @@ import json import os import shutil +from typing import Any import uuid from loguru import logger as log from fastapi import HTTPException, UploadFile @@ -477,21 +478,21 @@ async def get_all_tasks_for_project(project_id, db): return [str(result[0]) for result in results] -async def update_total_image_uploaded( - db: Connection, project_id: uuid.UUID, task_id: uuid.UUID, total_image_count: str +async def update_task_field( + db: Connection, project_id: uuid.UUID, task_id: uuid.UUID, column: Any, value: str ): """ - Update the total_image_uploaded field in the tasks table. + Generic function to update a field(assets_url and total_image_count) in the tasks table. """ async with db.cursor() as cur: await cur.execute( - """ + f""" UPDATE tasks - SET total_image_uploaded = %(total_image_uploaded)s + SET {column} = %(value)s WHERE project_id = %(project_id)s AND id = %(task_id)s; """, { - "total_image_uploaded": total_image_count, + "value": value, "project_id": str(project_id), "task_id": str(task_id), }, diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 69239db3..73aaf67d 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -25,7 +25,7 @@ ) from psycopg.rows import dict_row from app.config import settings -from app.s3 import get_presigned_url +from app.s3 import generate_static_url, get_presigned_url class CentroidOut(BaseModel): @@ -179,6 +179,16 @@ class TaskOut(BaseModel): total_area_sqkm: Optional[float] = None flight_time_minutes: Optional[float] = None flight_distance_km: Optional[float] = None + total_image_uploaded: Optional[int] = None + + @model_validator(mode="after") + def set_assets_url(cls, values): + """Set image_url before rendering the model.""" + assets_url = values.assets_url + if assets_url: + values.assets_url = generate_static_url(settings.S3_BUCKET_NAME, assets_url) + + return values class DbProject(BaseModel): @@ -295,6 +305,8 @@ async def one(db: Connection, project_id: uuid.UUID): t.total_area_sqkm, t.flight_time_minutes, t.flight_distance_km, + t.assets_url, + t.total_image_uploaded, ST_AsGeoJSON(t.outline)::jsonb -> 'coordinates' AS coordinates, ST_AsGeoJSON(t.outline)::jsonb -> 'type' AS type, ST_XMin(ST_Envelope(t.outline)) AS xmin, @@ -323,6 +335,8 @@ async def one(db: Connection, project_id: uuid.UUID): total_area_sqkm, flight_distance_km, flight_time_minutes, + total_image_uploaded, + assets_url, jsonb_build_object( 'type', 'Feature', 'geometry', jsonb_build_object( diff --git a/src/backend/app/s3.py b/src/backend/app/s3.py index 22d83113..afdc3778 100644 --- a/src/backend/app/s3.py +++ b/src/backend/app/s3.py @@ -4,6 +4,7 @@ from io import BytesIO from typing import Any from datetime import timedelta +from urllib.parse import urljoin def s3_client(): @@ -215,19 +216,9 @@ def get_object_metadata(bucket_name: str, object_name: str): return client.stat_object(bucket_name, object_name) -def get_cog_path(bucket_name: str, project_id: str, task_id: str): - """Generate the presigned URL for a COG file in an S3 bucket. - - Args: - bucket_name (str): The name of the S3 bucket. - project_id (str): The unique project identifier. - orthophoto_name (str): The name of the COG file. - - Returns: - str: The presigned URL to access the COG file. - """ - # Construct the S3 path for the COG file - s3_path = f"dtm-data/projects/{project_id}/{task_id}/orthophoto/odm_orthophoto.tif" - - # Get the presigned URL - return get_presigned_url(bucket_name, s3_path) +def generate_static_url(bucket_name: str, s3_path: str): + """Generate a static URL for an S3 object.""" + minio_url, is_secure = is_connection_secure(settings.S3_ENDPOINT) + protocol = "https" if is_secure else "http" + base_url = f"{protocol}://{minio_url}/{bucket_name}/" + return urljoin(base_url, s3_path) diff --git a/src/backend/app/tasks/task_logic.py b/src/backend/app/tasks/task_logic.py index 57215849..d23777da 100644 --- a/src/backend/app/tasks/task_logic.py +++ b/src/backend/app/tasks/task_logic.py @@ -608,8 +608,8 @@ async def handle_event( project_id, task_id ).image_count - await project_logic.update_total_image_uploaded( - db, project_id, task_id, toatl_image_count + await project_logic.update_task_field( + db, project_id, task_id, "total_image_uploaded", toatl_image_count ) return await update_task_state( diff --git a/src/backend/app/tasks/task_schemas.py b/src/backend/app/tasks/task_schemas.py index 286fa7b4..8dc57708 100644 --- a/src/backend/app/tasks/task_schemas.py +++ b/src/backend/app/tasks/task_schemas.py @@ -9,7 +9,7 @@ from psycopg.rows import class_row, dict_row from typing import List, Literal, Optional from pydantic.functional_validators import field_validator -from app.s3 import is_connection_secure +from app.s3 import generate_static_url, is_connection_secure class Geometry(BaseModel): @@ -270,6 +270,8 @@ class TaskDetailsOut(BaseModel): total_area_sqkm: Optional[float] = None flight_time_minutes: Optional[float] = None flight_distance_km: Optional[float] = None + total_image_uploaded: Optional[int] = None + assets_url: Optional[str] = None outline: Outline created_at: datetime updated_at: Optional[datetime] = None @@ -282,6 +284,15 @@ class TaskDetailsOut(BaseModel): gimble_angles_degrees: Optional[int] = None centroid: dict + @model_validator(mode="after") + def set_assets_url(cls, values): + """Set image_url before rendering the model.""" + assets_url = values.assets_url + if assets_url: + values.assets_url = generate_static_url(settings.S3_BUCKET_NAME, assets_url) + + return values + @field_validator("state", mode="after") @classmethod def integer_state_to_string(cls, value: State): @@ -309,7 +320,8 @@ async def get_task_details(db: Connection, task_id: uuid.UUID): tasks.total_area_sqkm, tasks.flight_time_minutes, tasks.flight_distance_km, - + tasks.total_image_uploaded, + tasks.assets_url, -- Construct the outline as a GeoJSON Feature jsonb_build_object( 'type', 'Feature', From b2919718df62df39def5614dc85d402de109e38e Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Mon, 30 Dec 2024 19:18:53 +0545 Subject: [PATCH 11/36] fix: run pre-commit for format migartions file --- .../app/migrations/versions/b18103ac4ab7_.py | 71 ++++++++++++++----- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/src/backend/app/migrations/versions/b18103ac4ab7_.py b/src/backend/app/migrations/versions/b18103ac4ab7_.py index fc117ba5..54ddb74c 100644 --- a/src/backend/app/migrations/versions/b18103ac4ab7_.py +++ b/src/backend/app/migrations/versions/b18103ac4ab7_.py @@ -5,6 +5,7 @@ Create Date: 2024-12-30 11:36:29.762485 """ + from typing import Sequence, Union from alembic import op @@ -12,33 +13,67 @@ from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision: str = 'b18103ac4ab7' -down_revision: Union[str, None] = 'e23c05f21542' +revision: str = "b18103ac4ab7" +down_revision: Union[str, None] = "e23c05f21542" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('task_events', 'state', - existing_type=postgresql.ENUM('REQUEST_FOR_MAPPING', 'UNLOCKED_TO_MAP', 'LOCKED_FOR_MAPPING', 'UNLOCKED_TO_VALIDATE', 'LOCKED_FOR_VALIDATION', 'UNLOCKED_DONE', 'UNFLYABLE_TASK', 'IMAGE_UPLOADED', 'IMAGE_PROCESSING_FAILED', 'IMAGE_PROCESSING_STARTED', 'IMAGE_PROCESSING_FINISHED', name='state'), - nullable=False) - op.add_column('tasks', sa.Column('total_area_sqkm', sa.Float(), nullable=True)) - op.add_column('tasks', sa.Column('flight_time_minutes', sa.Float(), nullable=True)) - op.add_column('tasks', sa.Column('flight_distance_km', sa.Float(), nullable=True)) - op.add_column('tasks', sa.Column('total_image_uploaded', sa.SmallInteger(), nullable=True)) - op.add_column('tasks', sa.Column('assets_url', sa.String(), nullable=True)) + op.alter_column( + "task_events", + "state", + existing_type=postgresql.ENUM( + "REQUEST_FOR_MAPPING", + "UNLOCKED_TO_MAP", + "LOCKED_FOR_MAPPING", + "UNLOCKED_TO_VALIDATE", + "LOCKED_FOR_VALIDATION", + "UNLOCKED_DONE", + "UNFLYABLE_TASK", + "IMAGE_UPLOADED", + "IMAGE_PROCESSING_FAILED", + "IMAGE_PROCESSING_STARTED", + "IMAGE_PROCESSING_FINISHED", + name="state", + ), + nullable=False, + ) + op.add_column("tasks", sa.Column("total_area_sqkm", sa.Float(), nullable=True)) + op.add_column("tasks", sa.Column("flight_time_minutes", sa.Float(), nullable=True)) + op.add_column("tasks", sa.Column("flight_distance_km", sa.Float(), nullable=True)) + op.add_column( + "tasks", sa.Column("total_image_uploaded", sa.SmallInteger(), nullable=True) + ) + op.add_column("tasks", sa.Column("assets_url", sa.String(), nullable=True)) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('tasks', 'assets_url') - op.drop_column('tasks', 'total_image_uploaded') - op.drop_column('tasks', 'flight_distance_km') - op.drop_column('tasks', 'flight_time_minutes') - op.drop_column('tasks', 'total_area_sqkm') - op.alter_column('task_events', 'state', - existing_type=postgresql.ENUM('REQUEST_FOR_MAPPING', 'UNLOCKED_TO_MAP', 'LOCKED_FOR_MAPPING', 'UNLOCKED_TO_VALIDATE', 'LOCKED_FOR_VALIDATION', 'UNLOCKED_DONE', 'UNFLYABLE_TASK', 'IMAGE_UPLOADED', 'IMAGE_PROCESSING_FAILED', 'IMAGE_PROCESSING_STARTED', 'IMAGE_PROCESSING_FINISHED', name='state'), - nullable=True) + op.drop_column("tasks", "assets_url") + op.drop_column("tasks", "total_image_uploaded") + op.drop_column("tasks", "flight_distance_km") + op.drop_column("tasks", "flight_time_minutes") + op.drop_column("tasks", "total_area_sqkm") + op.alter_column( + "task_events", + "state", + existing_type=postgresql.ENUM( + "REQUEST_FOR_MAPPING", + "UNLOCKED_TO_MAP", + "LOCKED_FOR_MAPPING", + "UNLOCKED_TO_VALIDATE", + "LOCKED_FOR_VALIDATION", + "UNLOCKED_DONE", + "UNFLYABLE_TASK", + "IMAGE_UPLOADED", + "IMAGE_PROCESSING_FAILED", + "IMAGE_PROCESSING_STARTED", + "IMAGE_PROCESSING_FINISHED", + name="state", + ), + nullable=True, + ) # ### end Alembic commands ### From 8672efe2ef2119a607a4da667198f91ce50ead0d Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Tue, 31 Dec 2024 09:42:00 +0545 Subject: [PATCH 12/36] fix: process assests from odm, download issues --- src/backend/app/projects/image_processing.py | 29 +++++++++++++------- src/backend/app/projects/project_logic.py | 2 +- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/backend/app/projects/image_processing.py b/src/backend/app/projects/image_processing.py index ff5f7ba5..03cc7cf0 100644 --- a/src/backend/app/projects/image_processing.py +++ b/src/backend/app/projects/image_processing.py @@ -363,13 +363,19 @@ async def process_assets_from_odm( """ log.info(f"Starting processing for project {dtm_project_id}") node = Node.from_url(node_odm_url) - output_file_path = f"/tmp/{dtm_project_id}" + output_file_path = f"/tmp/{uuid.uuid4()}" try: + os.makedirs(output_file_path, exist_ok=True) task = node.get_task(odm_task_id) - log.info(f"Downloading results for task {dtm_project_id} to {output_file_path}") + log.info(f"Downloading results for task {odm_task_id} to {output_file_path}") assets_path = task.download_zip(output_file_path) + if not os.path.exists(assets_path): + log.error(f"Downloaded file not found: {assets_path}") + raise + log.info(f"Successfully downloaded ZIP to {assets_path}") + s3_path = f"dtm-data/projects/{dtm_project_id}/{dtm_task_id if dtm_task_id else ''}/assets.zip".strip( "/" ) @@ -395,14 +401,16 @@ async def process_assets_from_odm( add_file_to_bucket(settings.S3_BUCKET_NAME, orthophoto_path, s3_ortho_path) images_json_path = os.path.join(output_file_path, "images.json") - s3_images_json_path = f"dtm-data/projects/{dtm_project_id}/{dtm_task_id if dtm_task_id else ''}/images.json".strip( - "/" - ) - - log.info(f"Uploading images.json to S3 path: {s3_images_json_path}") - add_file_to_bucket( - settings.S3_BUCKET_NAME, images_json_path, s3_images_json_path - ) + if os.path.exists(images_json_path): + s3_images_json_path = f"dtm-data/projects/{dtm_project_id}/{dtm_task_id if dtm_task_id else ''}/images.json".strip( + "/" + ) + log.info(f"Uploading images.json to S3 path: {s3_images_json_path}") + add_file_to_bucket( + settings.S3_BUCKET_NAME, images_json_path, s3_images_json_path + ) + else: + log.warning(f"images.json not found in {output_file_path}") log.info(f"Processing complete for project {dtm_project_id}") @@ -425,6 +433,7 @@ async def process_assets_from_odm( log.info( f"Task {dtm_task_id} state updated to IMAGE_PROCESSING_FINISHED in the database." ) + s3_path_url = ( f"dtm-data/projects/{dtm_project_id}/{dtm_task_id}/assets.zip" ) diff --git a/src/backend/app/projects/project_logic.py b/src/backend/app/projects/project_logic.py index f20326ca..4eac3969 100644 --- a/src/backend/app/projects/project_logic.py +++ b/src/backend/app/projects/project_logic.py @@ -51,7 +51,7 @@ async def get_centroids(db: Connection): p.name, ST_AsGeoJSON(p.centroid)::jsonb AS centroid, COUNT(t.id) AS total_task_count, - COUNT(CASE WHEN te.state IN ('LOCKED_FOR_MAPPING', 'REQUEST_FOR_MAPPING', 'IMAGE_UPLOADED', 'UNFLYABLE_TASK') THEN 1 END) AS ongoing_task_count, + COUNT(CASE WHEN te.state IN ('LOCKED_FOR_MAPPING', 'REQUEST_FOR_MAPPING', 'IMAGE_UPLOADED', 'UNFLYABLE_TASK', 'IMAGE_PROCESSING_STARTED') THEN 1 END) AS ongoing_task_count, COUNT(CASE WHEN te.state = 'IMAGE_PROCESSING_FINISHED' THEN 1 END) AS completed_task_count FROM projects p From 89f1fb385caeea90c60955bffce44620688d26c1 Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Thu, 26 Dec 2024 13:53:34 +0545 Subject: [PATCH 13/36] feat: update the gcp files for all project images --- src/backend/app/gcp/gcp_routes.py | 12 ++++++------ src/backend/app/projects/image_processing.py | 7 +++++++ src/backend/app/projects/project_routes.py | 13 ++++++++++++- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/backend/app/gcp/gcp_routes.py b/src/backend/app/gcp/gcp_routes.py index 31799e50..bc68cdba 100644 --- a/src/backend/app/gcp/gcp_routes.py +++ b/src/backend/app/gcp/gcp_routes.py @@ -1,5 +1,6 @@ import uuid from app.config import settings +from app.projects import project_schemas from fastapi import APIRouter, Depends from app.waypoints import waypoint_schemas from app.gcp import gcp_crud @@ -21,15 +22,15 @@ async def find_images( project_id: uuid.UUID, task_id: uuid.UUID, + db: Annotated[Connection, Depends(database.get_db)], point: waypoint_schemas.PointField = None, ) -> List[str]: """Find images that contain a specified point.""" fov_degree = 82.1 # For DJI Mini 4 Pro - altitude = 100 # TODO: Get this from db - + result = await project_schemas.DbProject.one(db, project_id) return await gcp_crud.find_images_in_a_task_for_point( - project_id, task_id, point, fov_degree, altitude + project_id, task_id, point, fov_degree, result.altitude ) @@ -42,11 +43,10 @@ async def find_images_for_a_project( """Find images that contain a specified point in a project.""" fov_degree = 82.1 # For DJI Mini 4 Pro - altitude = 100 # TODO: Get this from db - + result = await project_schemas.DbProject.one(db, project_id) # Get all task IDs for the project from database task_id_list = await list_task_id_for_project(db, project_id) return await gcp_crud.find_images_in_a_project_for_point( - project_id, task_id_list, point, fov_degree, altitude + project_id, task_id_list, point, fov_degree, result.altitude ) diff --git a/src/backend/app/projects/image_processing.py b/src/backend/app/projects/image_processing.py index ec0a2de7..e7757486 100644 --- a/src/backend/app/projects/image_processing.py +++ b/src/backend/app/projects/image_processing.py @@ -165,6 +165,13 @@ async def _process_images( self.download_images_from_s3(bucket_name, temp_dir, self.task_id) images_list = self.list_images(temp_dir) else: + gcp_list_file = f"dtm-data/projects/{self.project_id}/gcp/gcp_list.txt" + gcp_file_path = os.path.join(temp_dir, "gcp_list.txt") + + # Check and add the GCP file to the images list if it exists + if get_file_from_bucket(bucket_name, gcp_list_file, gcp_file_path): + images_list.append(gcp_file_path) + for task_id in self.task_ids: self.download_images_from_s3(bucket_name, temp_dir, task_id) images_list.extend(self.list_images(temp_dir)) diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index cc9e43c1..cfe23cc8 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -467,12 +467,23 @@ async def process_all_imagery( user_data: Annotated[AuthUser, Depends(login_required)], background_tasks: BackgroundTasks, db: Annotated[Connection, Depends(database.get_db)], + gcp_file: UploadFile = File(None), + ): """ API endpoint to process all tasks associated with a project. """ user_id = user_data.id - + if gcp_file: + gcp_file_path = f"/tmp/{uuid.uuid4()}" + with open(gcp_file_path, "wb") as f: + f.write(await gcp_file.read()) + + s3_path = ( + f"dtm-data/projects/{project_id}/gcp/gcp_list.txt" + ) + add_file_to_bucket( settings.S3_BUCKET_NAME, gcp_file_path, s3_path) + tasks = await project_logic.get_all_tasks_for_project(project.id, db) background_tasks.add_task( project_logic.process_all_drone_images, project_id, tasks, user_id, db From f5295657e94e9db24e1bb80e05318476a717b96f Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Thu, 26 Dec 2024 14:13:15 +0545 Subject: [PATCH 14/36] feat: update the new fields in tasks tables --- src/backend/app/db/db_models.py | 4 ++ .../app/migrations/versions/d202ea539f6d_.py | 42 +++++++++++++++++++ src/backend/app/projects/image_processing.py | 2 +- src/backend/app/projects/project_logic.py | 30 +++++++++++-- src/backend/app/projects/project_routes.py | 11 ++--- 5 files changed, 77 insertions(+), 12 deletions(-) create mode 100644 src/backend/app/migrations/versions/d202ea539f6d_.py diff --git a/src/backend/app/db/db_models.py b/src/backend/app/db/db_models.py index 8b4a2ac7..8d8f159e 100644 --- a/src/backend/app/db/db_models.py +++ b/src/backend/app/db/db_models.py @@ -78,6 +78,10 @@ class DbTask(Base): take_off_point = cast( WKBElement, Column(Geometry("POINT", srid=4326), nullable=True) ) + total_area_sqkm = cast(float, Column(Float, nullable=True)) + flight_time_minutes = cast(int, Column(Float, nullable=True)) + flight_distance_km = cast(float, Column(Float, nullable=True)) + total_image_uploaded = cast(int, Column(SmallInteger, nullable=True)) class DbProject(Base): diff --git a/src/backend/app/migrations/versions/d202ea539f6d_.py b/src/backend/app/migrations/versions/d202ea539f6d_.py new file mode 100644 index 00000000..f6107002 --- /dev/null +++ b/src/backend/app/migrations/versions/d202ea539f6d_.py @@ -0,0 +1,42 @@ +""" + +Revision ID: d202ea539f6d +Revises: e23c05f21542 +Create Date: 2024-12-26 08:11:00.011691 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'd202ea539f6d' +down_revision: Union[str, None] = 'e23c05f21542' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('task_events', 'state', + existing_type=postgresql.ENUM('REQUEST_FOR_MAPPING', 'UNLOCKED_TO_MAP', 'LOCKED_FOR_MAPPING', 'UNLOCKED_TO_VALIDATE', 'LOCKED_FOR_VALIDATION', 'UNLOCKED_DONE', 'UNFLYABLE_TASK', 'IMAGE_UPLOADED', 'IMAGE_PROCESSING_FAILED', 'IMAGE_PROCESSING_STARTED', 'IMAGE_PROCESSING_FINISHED', name='state'), + nullable=False) + op.add_column('tasks', sa.Column('total_area_sqkm', sa.Float(), nullable=True)) + op.add_column('tasks', sa.Column('flight_time_minutes', sa.Float(), nullable=True)) + op.add_column('tasks', sa.Column('flight_distance_km', sa.Float(), nullable=True)) + op.add_column('tasks', sa.Column('total_image_uploaded', sa.SmallInteger(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('tasks', 'total_image_uploaded') + op.drop_column('tasks', 'flight_distance_km') + op.drop_column('tasks', 'flight_time_minutes') + op.drop_column('tasks', 'total_area_sqkm') + op.alter_column('task_events', 'state', + existing_type=postgresql.ENUM('REQUEST_FOR_MAPPING', 'UNLOCKED_TO_MAP', 'LOCKED_FOR_MAPPING', 'UNLOCKED_TO_VALIDATE', 'LOCKED_FOR_VALIDATION', 'UNLOCKED_DONE', 'UNFLYABLE_TASK', 'IMAGE_UPLOADED', 'IMAGE_PROCESSING_FAILED', 'IMAGE_PROCESSING_STARTED', 'IMAGE_PROCESSING_FINISHED', name='state'), + nullable=True) + # ### end Alembic commands ### diff --git a/src/backend/app/projects/image_processing.py b/src/backend/app/projects/image_processing.py index e7757486..d9a4ead2 100644 --- a/src/backend/app/projects/image_processing.py +++ b/src/backend/app/projects/image_processing.py @@ -171,7 +171,7 @@ async def _process_images( # Check and add the GCP file to the images list if it exists if get_file_from_bucket(bucket_name, gcp_list_file, gcp_file_path): images_list.append(gcp_file_path) - + for task_id in self.task_ids: self.download_images_from_s3(bucket_name, temp_dir, task_id) images_list.extend(self.list_images(temp_dir)) diff --git a/src/backend/app/projects/project_logic.py b/src/backend/app/projects/project_logic.py index af1769ad..184a032e 100644 --- a/src/backend/app/projects/project_logic.py +++ b/src/backend/app/projects/project_logic.py @@ -150,10 +150,11 @@ async def create_tasks_from_geojson( async with db.cursor() as cur: await cur.execute( """ - INSERT INTO tasks (id, project_id, outline, project_task_index) - VALUES (%(id)s, %(project_id)s, %(outline)s, %(project_task_index)s) - RETURNING id; - """, + INSERT INTO tasks (id, project_id, outline, project_task_index, total_area_sqkm) + VALUES (%(id)s, %(project_id)s, %(outline)s, %(project_task_index)s, + ST_Area(ST_Transform(ST_SetSRID(outline, 4326), 3857)) / 1000000) + RETURNING id; + """, { "id": task_id, "project_id": project_id, @@ -163,7 +164,28 @@ async def create_tasks_from_geojson( "project_task_index": index + 1, }, ) + + # async with db.cursor() as cur: + # await cur.execute( + # """ + # INSERT INTO tasks (id, project_id, outline, project_task_index) + # VALUES (%(id)s, %(project_id)s, %(outline)s, %(project_task_index)s) + # RETURNING id; + # ST_Area(ST_Transform(t.outline, 3857)) / 1000000 AS task_area + + # """, + # { + # "id": task_id, + # "project_id": project_id, + # "outline": wkblib.dumps( + # shape(polygon["geometry"]), hex=True + # ), + # "project_task_index": index + 1, + # }, + # ) + result = await cur.fetchone() + if result: log.debug( "Created database task | " diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index cfe23cc8..ce4c06f5 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -468,7 +468,6 @@ async def process_all_imagery( background_tasks: BackgroundTasks, db: Annotated[Connection, Depends(database.get_db)], gcp_file: UploadFile = File(None), - ): """ API endpoint to process all tasks associated with a project. @@ -478,12 +477,10 @@ async def process_all_imagery( gcp_file_path = f"/tmp/{uuid.uuid4()}" with open(gcp_file_path, "wb") as f: f.write(await gcp_file.read()) - - s3_path = ( - f"dtm-data/projects/{project_id}/gcp/gcp_list.txt" - ) - add_file_to_bucket( settings.S3_BUCKET_NAME, gcp_file_path, s3_path) - + + s3_path = f"dtm-data/projects/{project_id}/gcp/gcp_list.txt" + add_file_to_bucket(settings.S3_BUCKET_NAME, gcp_file_path, s3_path) + tasks = await project_logic.get_all_tasks_for_project(project.id, db) background_tasks.add_task( project_logic.process_all_drone_images, project_id, tasks, user_id, db From bb76da0ab5bc7a6d233c3bcc4b4766cc623d90bc Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Thu, 26 Dec 2024 15:13:44 +0545 Subject: [PATCH 15/36] feat: update task areas from db to instead of postgis --- .../app/migrations/versions/d202ea539f6d_.py | 67 ++++++++++++++----- src/backend/app/projects/project_logic.py | 55 ++++++++------- src/backend/app/projects/project_schemas.py | 9 ++- src/backend/app/tasks/task_schemas.py | 8 +-- .../Dashboard/TaskLogs/TaskLogsTable.tsx | 2 +- .../DescriptionBox/index.tsx | 4 +- .../Tasks/TableSection/index.tsx | 4 +- 7 files changed, 91 insertions(+), 58 deletions(-) diff --git a/src/backend/app/migrations/versions/d202ea539f6d_.py b/src/backend/app/migrations/versions/d202ea539f6d_.py index f6107002..9830f62e 100644 --- a/src/backend/app/migrations/versions/d202ea539f6d_.py +++ b/src/backend/app/migrations/versions/d202ea539f6d_.py @@ -5,6 +5,7 @@ Create Date: 2024-12-26 08:11:00.011691 """ + from typing import Sequence, Union from alembic import op @@ -12,31 +13,65 @@ from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision: str = 'd202ea539f6d' -down_revision: Union[str, None] = 'e23c05f21542' +revision: str = "d202ea539f6d" +down_revision: Union[str, None] = "e23c05f21542" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('task_events', 'state', - existing_type=postgresql.ENUM('REQUEST_FOR_MAPPING', 'UNLOCKED_TO_MAP', 'LOCKED_FOR_MAPPING', 'UNLOCKED_TO_VALIDATE', 'LOCKED_FOR_VALIDATION', 'UNLOCKED_DONE', 'UNFLYABLE_TASK', 'IMAGE_UPLOADED', 'IMAGE_PROCESSING_FAILED', 'IMAGE_PROCESSING_STARTED', 'IMAGE_PROCESSING_FINISHED', name='state'), - nullable=False) - op.add_column('tasks', sa.Column('total_area_sqkm', sa.Float(), nullable=True)) - op.add_column('tasks', sa.Column('flight_time_minutes', sa.Float(), nullable=True)) - op.add_column('tasks', sa.Column('flight_distance_km', sa.Float(), nullable=True)) - op.add_column('tasks', sa.Column('total_image_uploaded', sa.SmallInteger(), nullable=True)) + op.alter_column( + "task_events", + "state", + existing_type=postgresql.ENUM( + "REQUEST_FOR_MAPPING", + "UNLOCKED_TO_MAP", + "LOCKED_FOR_MAPPING", + "UNLOCKED_TO_VALIDATE", + "LOCKED_FOR_VALIDATION", + "UNLOCKED_DONE", + "UNFLYABLE_TASK", + "IMAGE_UPLOADED", + "IMAGE_PROCESSING_FAILED", + "IMAGE_PROCESSING_STARTED", + "IMAGE_PROCESSING_FINISHED", + name="state", + ), + nullable=False, + ) + op.add_column("tasks", sa.Column("total_area_sqkm", sa.Float(), nullable=True)) + op.add_column("tasks", sa.Column("flight_time_minutes", sa.Float(), nullable=True)) + op.add_column("tasks", sa.Column("flight_distance_km", sa.Float(), nullable=True)) + op.add_column( + "tasks", sa.Column("total_image_uploaded", sa.SmallInteger(), nullable=True) + ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('tasks', 'total_image_uploaded') - op.drop_column('tasks', 'flight_distance_km') - op.drop_column('tasks', 'flight_time_minutes') - op.drop_column('tasks', 'total_area_sqkm') - op.alter_column('task_events', 'state', - existing_type=postgresql.ENUM('REQUEST_FOR_MAPPING', 'UNLOCKED_TO_MAP', 'LOCKED_FOR_MAPPING', 'UNLOCKED_TO_VALIDATE', 'LOCKED_FOR_VALIDATION', 'UNLOCKED_DONE', 'UNFLYABLE_TASK', 'IMAGE_UPLOADED', 'IMAGE_PROCESSING_FAILED', 'IMAGE_PROCESSING_STARTED', 'IMAGE_PROCESSING_FINISHED', name='state'), - nullable=True) + op.drop_column("tasks", "total_image_uploaded") + op.drop_column("tasks", "flight_distance_km") + op.drop_column("tasks", "flight_time_minutes") + op.drop_column("tasks", "total_area_sqkm") + op.alter_column( + "task_events", + "state", + existing_type=postgresql.ENUM( + "REQUEST_FOR_MAPPING", + "UNLOCKED_TO_MAP", + "LOCKED_FOR_MAPPING", + "UNLOCKED_TO_VALIDATE", + "LOCKED_FOR_VALIDATION", + "UNLOCKED_DONE", + "UNFLYABLE_TASK", + "IMAGE_UPLOADED", + "IMAGE_PROCESSING_FAILED", + "IMAGE_PROCESSING_STARTED", + "IMAGE_PROCESSING_FINISHED", + name="state", + ), + nullable=True, + ) # ### end Alembic commands ### diff --git a/src/backend/app/projects/project_logic.py b/src/backend/app/projects/project_logic.py index 184a032e..473e4efb 100644 --- a/src/backend/app/projects/project_logic.py +++ b/src/backend/app/projects/project_logic.py @@ -2,7 +2,6 @@ import uuid from loguru import logger as log from fastapi import HTTPException, UploadFile -from pyproj import Transformer from app.tasks.task_splitter import split_by_square from fastapi.concurrency import run_in_threadpool from psycopg import Connection @@ -21,6 +20,8 @@ from app.projects import project_schemas from minio import S3Error from psycopg.rows import dict_row +from shapely.ops import transform +import pyproj async def get_centroids(db: Connection): @@ -127,12 +128,20 @@ async def create_tasks_from_geojson( if isinstance(boundaries, str): boundaries = json.loads(boundaries) - # Update the boundary polyon on the database. if boundaries["type"] == "Feature": polygons = [boundaries] else: polygons = boundaries["features"] + log.debug(f"Processing {len(polygons)} task geometries") + + # Set up the projection transform for EPSG:3857 (Web Mercator) + proj_wgs84 = pyproj.CRS("EPSG:4326") + proj_mercator = pyproj.CRS("EPSG:3857") + project_transformer = pyproj.Transformer.from_crs( + proj_wgs84, proj_mercator, always_xy=True + ) + for index, polygon in enumerate(polygons): try: if not polygon["geometry"]: @@ -141,18 +150,25 @@ async def create_tasks_from_geojson( if polygon["geometry"]["type"] == "MultiPolygon": log.debug("Converting MultiPolygon to Polygon") polygon["geometry"]["type"] = "Polygon" - polygon["geometry"]["coordinates"] = polygon["geometry"][ "coordinates" ][0] + geom = shape(polygon["geometry"]) + + # Transform the geometry to EPSG:3857 and calculate the area in square meters + transformed_geom = transform(project_transformer.transform, geom) + area_sq_m = transformed_geom.area # Area in square meters + + # Convert area to square kilometers + total_area_sqkm = area_sq_m / 1_000_000 + task_id = str(uuid.uuid4()) async with db.cursor() as cur: await cur.execute( """ INSERT INTO tasks (id, project_id, outline, project_task_index, total_area_sqkm) - VALUES (%(id)s, %(project_id)s, %(outline)s, %(project_task_index)s, - ST_Area(ST_Transform(ST_SetSRID(outline, 4326), 3857)) / 1000000) + VALUES (%(id)s, %(project_id)s, %(outline)s, %(project_task_index)s, %(total_area_sqkm)s) RETURNING id; """, { @@ -162,30 +178,11 @@ async def create_tasks_from_geojson( shape(polygon["geometry"]), hex=True ), "project_task_index": index + 1, + "total_area_sqkm": total_area_sqkm, }, ) - - # async with db.cursor() as cur: - # await cur.execute( - # """ - # INSERT INTO tasks (id, project_id, outline, project_task_index) - # VALUES (%(id)s, %(project_id)s, %(outline)s, %(project_task_index)s) - # RETURNING id; - # ST_Area(ST_Transform(t.outline, 3857)) / 1000000 AS task_area - - # """, - # { - # "id": task_id, - # "project_id": project_id, - # "outline": wkblib.dumps( - # shape(polygon["geometry"]), hex=True - # ), - # "project_task_index": index + 1, - # }, - # ) - result = await cur.fetchone() - + if result: log.debug( "Created database task | " @@ -342,8 +339,10 @@ async def check_regulator_project(db: Connection, project_id: str, email: str): def generate_square_geojson(center_lat, center_lon, side_length_meters): - transformer = Transformer.from_crs("EPSG:4326", "EPSG:3857", always_xy=True) - transformer_back = Transformer.from_crs("EPSG:3857", "EPSG:4326", always_xy=True) + transformer = pyproj.Transformer.from_crs("EPSG:4326", "EPSG:3857", always_xy=True) + transformer_back = pyproj.Transformer.from_crs( + "EPSG:3857", "EPSG:4326", always_xy=True + ) center_x, center_y = transformer.transform(center_lon, center_lat) half_side = side_length_meters / 2 diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 962b2f4c..09be3fae 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -173,10 +173,10 @@ class TaskOut(BaseModel): outline: Optional[Polygon | Feature | FeatureCollection] = None state: Optional[str] = None user_id: Optional[str] = None - task_area: Optional[float] = None name: Optional[str] = None image_count: Optional[int] = None assets_url: Optional[str] = None + total_area_sqkm: Optional[float] = None class DbProject(BaseModel): @@ -210,7 +210,6 @@ class DbProject(BaseModel): is_terrain_follow: bool = False image_url: Optional[str] = None created_at: datetime - author_id: str async def one(db: Connection, project_id: uuid.UUID): """Get a single project & all associated tasks by ID.""" @@ -291,6 +290,7 @@ async def one(db: Connection, project_id: uuid.UUID): t.id, t.project_task_index, t.project_id, + t.total_area_sqkm, ST_AsGeoJSON(t.outline)::jsonb -> 'coordinates' AS coordinates, ST_AsGeoJSON(t.outline)::jsonb -> 'type' AS type, ST_XMin(ST_Envelope(t.outline)) AS xmin, @@ -299,8 +299,7 @@ async def one(db: Connection, project_id: uuid.UUID): ST_YMax(ST_Envelope(t.outline)) AS ymax, tsc.state AS state, tsc.user_id, - u.name, - ST_Area(ST_Transform(t.outline, 3857)) / 1000000 AS task_area + u.name FROM tasks t LEFT JOIN @@ -316,8 +315,8 @@ async def one(db: Connection, project_id: uuid.UUID): state, user_id, name, - task_area, project_id, + total_area_sqkm, jsonb_build_object( 'type', 'Feature', 'geometry', jsonb_build_object( diff --git a/src/backend/app/tasks/task_schemas.py b/src/backend/app/tasks/task_schemas.py index 31715c2e..9f66265e 100644 --- a/src/backend/app/tasks/task_schemas.py +++ b/src/backend/app/tasks/task_schemas.py @@ -165,7 +165,7 @@ async def all(db: Connection, project_id: uuid.UUID): class UserTasksStatsOut(BaseModel): task_id: uuid.UUID - task_area: float + total_area_sqkm: Optional[float] = None created_at: datetime state: str project_id: uuid.UUID @@ -207,7 +207,7 @@ async def get_tasks_by_user( tasks.project_task_index AS project_task_index, task_events.project_id AS project_id, projects.name AS project_name, - ST_Area(ST_Transform(tasks.outline, 3857)) / 1000000 AS task_area, + tasks.total_area_sqkm, task_events.created_at, task_events.updated_at, task_events.state, @@ -263,7 +263,7 @@ async def get_tasks_by_user( class TaskDetailsOut(BaseModel): - task_area: float + total_area_sqkm: float outline: Outline created_at: datetime updated_at: Optional[datetime] = None @@ -300,7 +300,7 @@ async def get_task_details(db: Connection, task_id: uuid.UUID): await cur.execute( """ SELECT - ST_Area(ST_Transform(tasks.outline, 3857)) / 1000000 AS task_area, + tasks.total_area_sqkm, -- Construct the outline as a GeoJSON Feature jsonb_build_object( diff --git a/src/frontend/src/components/Dashboard/TaskLogs/TaskLogsTable.tsx b/src/frontend/src/components/Dashboard/TaskLogs/TaskLogsTable.tsx index a46a31f2..77e5f007 100644 --- a/src/frontend/src/components/Dashboard/TaskLogs/TaskLogsTable.tsx +++ b/src/frontend/src/components/Dashboard/TaskLogs/TaskLogsTable.tsx @@ -42,7 +42,7 @@ const TaskLogsTable = ({ data: taskList }: ITaskLogsTableProps) => { {task?.project_name} - {Number(task?.task_area)?.toFixed(3)} + {Number(task?.total_area_sqkm)?.toFixed(3)} {/* - */} diff --git a/src/frontend/src/components/DroneOperatorTask/DescriptionSection/DescriptionBox/index.tsx b/src/frontend/src/components/DroneOperatorTask/DescriptionSection/DescriptionBox/index.tsx index d68273ee..bd6328c1 100644 --- a/src/frontend/src/components/DroneOperatorTask/DescriptionSection/DescriptionBox/index.tsx +++ b/src/frontend/src/components/DroneOperatorTask/DescriptionSection/DescriptionBox/index.tsx @@ -135,8 +135,8 @@ const DescriptionBox = () => { }, { name: 'Total task area', - value: taskData?.task_area - ? `${Number(taskData?.task_area)?.toFixed(3)} km²` + value: taskData?.total_area_sqkm + ? `${Number(taskData?.total_area_sqkm)?.toFixed(3)} km²` : null, }, { diff --git a/src/frontend/src/components/IndividualProject/Tasks/TableSection/index.tsx b/src/frontend/src/components/IndividualProject/Tasks/TableSection/index.tsx index 340ab791..22c3f20e 100644 --- a/src/frontend/src/components/IndividualProject/Tasks/TableSection/index.tsx +++ b/src/frontend/src/components/IndividualProject/Tasks/TableSection/index.tsx @@ -10,7 +10,7 @@ const tasksDataColumns = [ { header: 'Task Area in km²', accessorKey: 'task_area', - }, + } ]; interface ITableSectionProps { @@ -34,7 +34,7 @@ export default function TableSection({ { id: `Task# ${curr?.project_task_index}`, flight_time: curr?.flight_time || '-', - task_area: Number(curr?.task_area)?.toFixed(3), + task_area: Number(curr?.total_area_sqkm)?.toFixed(3), task_id: curr?.id, // status: curr?.state, }, From e4d92afc2088027ac5217500aecc0b7f4c88f26d Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Mon, 30 Dec 2024 13:05:32 +0545 Subject: [PATCH 16/36] feat: update task flight time, flight distance & task areas --- src/backend/app/models/enums.py | 2 +- src/backend/app/projects/project_logic.py | 113 +++++++++++++++++- src/backend/app/projects/project_routes.py | 7 +- src/backend/app/projects/project_schemas.py | 6 + src/backend/app/tasks/task_logic.py | 9 ++ src/backend/app/tasks/task_schemas.py | 11 +- src/backend/app/utils.py | 23 ++-- .../Tasks/TableSection/index.tsx | 10 ++ 8 files changed, 160 insertions(+), 21 deletions(-) diff --git a/src/backend/app/models/enums.py b/src/backend/app/models/enums.py index 76d54c8d..cb6cebad 100644 --- a/src/backend/app/models/enums.py +++ b/src/backend/app/models/enums.py @@ -125,7 +125,7 @@ class DroneType(IntEnum): DJI_MINI_4_PRO = 1 -class UserRole(IntEnum, Enum): +class UserRole(int, Enum): PROJECT_CREATOR = 1 DRONE_PILOT = 2 REGULATOR = 3 diff --git a/src/backend/app/projects/project_logic.py b/src/backend/app/projects/project_logic.py index 473e4efb..a632a17b 100644 --- a/src/backend/app/projects/project_logic.py +++ b/src/backend/app/projects/project_logic.py @@ -22,6 +22,19 @@ from psycopg.rows import dict_row from shapely.ops import transform import pyproj +from geojson import Feature, FeatureCollection, Polygon +from app.s3 import get_file_from_bucket +from app.utils import ( + calculate_flight_time_from_placemarks, +) +import geojson +from drone_flightplan import ( + waypoints, + add_elevation_from_dem, + calculate_parameters, + create_placemarks, +) +from app.models.enums import FlightMode async def get_centroids(db: Connection): @@ -122,6 +135,7 @@ async def create_tasks_from_geojson( db: Connection, project_id: uuid.UUID, boundaries: str, + project: project_schemas.DbProject, ): """Create tasks for a project, from provided task boundaries.""" try: @@ -143,6 +157,77 @@ async def create_tasks_from_geojson( ) for index, polygon in enumerate(polygons): + forward_overlap = project.front_overlap if project.front_overlap else 70 + side_overlap = project.side_overlap if project.side_overlap else 70 + generate_3d = False # TODO: For 3d imageries drone_flightplan package needs to be updated. + + gsd = project.gsd_cm_px + altitude = project.altitude_from_ground + + parameters = calculate_parameters( + forward_overlap, + side_overlap, + altitude, + gsd, + 2, # Image Interval is set to 2 + ) + + # Wrap polygon into GeoJSON Feature + coordinates = polygon["geometry"]["coordinates"] + if polygon["geometry"]["type"] == "Polygon": + coordinates = polygon["geometry"]["coordinates"] + feature = Feature(geometry=Polygon(coordinates), properties={}) + feature_collection = FeatureCollection([feature]) + + # Common parameters for create_waypoint + waypoint_params = { + "project_area": feature_collection, + "agl": altitude, + "gsd": gsd, + "forward_overlap": forward_overlap, + "side_overlap": side_overlap, + "rotation_angle": 0, + "generate_3d": generate_3d, + } + waypoint_params["mode"] = FlightMode.waypoints + if project.is_terrain_follow: + dem_path = f"/tmp/{uuid.uuid4()}/dem.tif" + + # Terrain follow uses waypoints mode, waylines are generated later + points = waypoints.create_waypoint(**waypoint_params) + + try: + get_file_from_bucket( + settings.S3_BUCKET_NAME, + f"dtm-data/projects/{project.id}/dem.tif", + dem_path, + ) + # TODO: Do this with inmemory data + outfile_with_elevation = "/tmp/output_file_with_elevation.geojson" + add_elevation_from_dem(dem_path, points, outfile_with_elevation) + + inpointsfile = open(outfile_with_elevation, "r") + points_with_elevation = inpointsfile.read() + + except Exception: + points_with_elevation = points + + placemarks = create_placemarks( + geojson.loads(points_with_elevation), parameters + ) + + else: + points = waypoints.create_waypoint(**waypoint_params) + placemarks = create_placemarks(geojson.loads(points), parameters) + + flight_time_minutes = calculate_flight_time_from_placemarks(placemarks).get( + "total_flight_time" + ) + flight_distance_km = calculate_flight_time_from_placemarks(placemarks).get( + "flight_distance_km" + ) + print(f"Flight time: {flight_time_minutes} minutes") + print(f"Flight distance: {flight_distance_km} km") try: if not polygon["geometry"]: continue @@ -167,8 +252,8 @@ async def create_tasks_from_geojson( async with db.cursor() as cur: await cur.execute( """ - INSERT INTO tasks (id, project_id, outline, project_task_index, total_area_sqkm) - VALUES (%(id)s, %(project_id)s, %(outline)s, %(project_task_index)s, %(total_area_sqkm)s) + INSERT INTO tasks (id, project_id, outline, project_task_index, total_area_sqkm, flight_time_minutes, flight_distance_km) + VALUES (%(id)s, %(project_id)s, %(outline)s, %(project_task_index)s, %(total_area_sqkm)s, %(flight_time_minutes)s, %(flight_distance_km)s) RETURNING id; """, { @@ -179,6 +264,8 @@ async def create_tasks_from_geojson( ), "project_task_index": index + 1, "total_area_sqkm": total_area_sqkm, + "flight_time_minutes": flight_time_minutes, + "flight_distance_km": flight_distance_km, }, ) result = await cur.fetchone() @@ -383,3 +470,25 @@ async def get_all_tasks_for_project(project_id, db): results = await cur.fetchall() # Convert UUIDs to string return [str(result[0]) for result in results] + + +async def update_total_image_uploaded( + db: Connection, project_id: uuid.UUID, task_id: uuid.UUID, total_image_count: str +): + """ + Update the total_image_uploaded field in the tasks table. + """ + async with db.cursor() as cur: + await cur.execute( + """ + UPDATE tasks + SET total_image_uploaded = %(total_image_uploaded)s + WHERE project_id = %(project_id)s AND id = %(task_id)s; + """, + { + "total_image_uploaded": total_image_count, + "project_id": str(project_id), + "task_id": str(task_id), + }, + ) + return True diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index ce4c06f5..28f284d1 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -257,7 +257,7 @@ async def upload_project_task_boundaries( dict: JSON containing success message, project ID, and number of tasks. """ log.debug("Creating tasks for each polygon in project") - await project_logic.create_tasks_from_geojson(db, project.id, task_featcol) + await project_logic.create_tasks_from_geojson(db, project.id, task_featcol, project) return {"message": "Project Boundary Uploaded", "project_id": f"{project.id}"} @@ -305,9 +305,11 @@ async def preview_split_by_square( @router.post("/generate-presigned-url/", tags=["Image Upload"]) async def generate_presigned_url( + db: Annotated[Connection, Depends(database.get_db)], user: Annotated[AuthUser, Depends(login_required)], data: project_schemas.PresignedUrlRequest, replace_existing: bool = False, + ): """ Generate a pre-signed URL for uploading an image to S3 Bucket. @@ -366,7 +368,7 @@ async def generate_presigned_url( status_code=HTTPStatus.BAD_REQUEST, detail=f"Failed to delete existing image. {e}", ) - + # Generate a new pre-signed URL for the image upload url = client.get_presigned_url( "PUT", @@ -745,7 +747,6 @@ async def get_assets_info( if task_id is None: # Fetch all tasks associated with the project tasks = await project_deps.get_tasks_by_project_id(project.id, db) - results = [] for task in tasks: diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 09be3fae..69239db3 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -177,6 +177,8 @@ class TaskOut(BaseModel): image_count: Optional[int] = None assets_url: Optional[str] = None total_area_sqkm: Optional[float] = None + flight_time_minutes: Optional[float] = None + flight_distance_km: Optional[float] = None class DbProject(BaseModel): @@ -291,6 +293,8 @@ async def one(db: Connection, project_id: uuid.UUID): t.project_task_index, t.project_id, t.total_area_sqkm, + t.flight_time_minutes, + t.flight_distance_km, ST_AsGeoJSON(t.outline)::jsonb -> 'coordinates' AS coordinates, ST_AsGeoJSON(t.outline)::jsonb -> 'type' AS type, ST_XMin(ST_Envelope(t.outline)) AS xmin, @@ -317,6 +321,8 @@ async def one(db: Connection, project_id: uuid.UUID): name, project_id, total_area_sqkm, + flight_distance_km, + flight_time_minutes, jsonb_build_object( 'type', 'Feature', 'geometry', jsonb_build_object( diff --git a/src/backend/app/tasks/task_logic.py b/src/backend/app/tasks/task_logic.py index e7277dec..57215849 100644 --- a/src/backend/app/tasks/task_logic.py +++ b/src/backend/app/tasks/task_logic.py @@ -4,6 +4,7 @@ from app.tasks.task_schemas import NewEvent, TaskStats from app.users import user_schemas from app.utils import render_email_template, send_notification_email +from app.projects import project_logic from psycopg import Connection from app.models.enums import EventType, HTTPStatus, State, UserRole from fastapi import HTTPException, BackgroundTasks @@ -602,6 +603,14 @@ async def handle_event( status_code=403, detail="You cannot upload an image for this task as it is locked by another user.", ) + # update the count of the task to image uploaded. + toatl_image_count = project_logic.get_project_info_from_s3( + project_id, task_id + ).image_count + + await project_logic.update_total_image_uploaded( + db, project_id, task_id, toatl_image_count + ) return await update_task_state( db, diff --git a/src/backend/app/tasks/task_schemas.py b/src/backend/app/tasks/task_schemas.py index 9f66265e..5d10247a 100644 --- a/src/backend/app/tasks/task_schemas.py +++ b/src/backend/app/tasks/task_schemas.py @@ -166,6 +166,9 @@ async def all(db: Connection, project_id: uuid.UUID): class UserTasksStatsOut(BaseModel): task_id: uuid.UUID total_area_sqkm: Optional[float] = None + flight_time_minutes: Optional[float] = None + flight_distance_km: Optional[float] = None + outline: Outline created_at: datetime state: str project_id: uuid.UUID @@ -208,6 +211,8 @@ async def get_tasks_by_user( task_events.project_id AS project_id, projects.name AS project_name, tasks.total_area_sqkm, + tasks.flight_time_minutes, + tasks.flight_distance_km, task_events.created_at, task_events.updated_at, task_events.state, @@ -263,7 +268,9 @@ async def get_tasks_by_user( class TaskDetailsOut(BaseModel): - total_area_sqkm: float + total_area_sqkm: Optional[float] = None + flight_time_minutes: Optional[float] = None + flight_distance_km: Optional[float] = None outline: Outline created_at: datetime updated_at: Optional[datetime] = None @@ -301,6 +308,8 @@ async def get_task_details(db: Connection, task_id: uuid.UUID): """ SELECT tasks.total_area_sqkm, + tasks.flight_time_minutes, + tasks.flight_distance_km, -- Construct the outline as a GeoJSON Feature jsonb_build_object( diff --git a/src/backend/app/utils.py b/src/backend/app/utils.py index 41212919..f3512275 100644 --- a/src/backend/app/utils.py +++ b/src/backend/app/utils.py @@ -558,17 +558,19 @@ async def send_project_approval_email_to_regulator( def calculate_flight_time_from_placemarks(placemarks: Dict) -> Dict: """ - Calculate the total and average flight time based on placemarks and dynamically format the output. + Calculate the total and average flight time and total flight distance based on placemarks. Args: placemarks (Dict): GeoJSON-like data structure with flight plan. Returns: - Dict: Contains formatted total flight time and segment times. + Dict: Contains formatted total flight time, segment times, and total distance. """ total_time = 0 + total_distance = 0 features = placemarks["features"] transformer = Transformer.from_crs("EPSG:4326", "EPSG:3857", always_xy=True) + for i in range(1, len(features)): # Extract current and previous coordinates prev_coords = features[i - 1]["geometry"]["coordinates"][:2] @@ -581,22 +583,15 @@ def calculate_flight_time_from_placemarks(placemarks: Dict) -> Dict: # Calculate distance (meters) and time (seconds) distance = prev_point.distance(curr_point) + total_distance += distance # Accumulate total distance segment_time = distance / speed total_time += segment_time - # Dynamically format the total flight time - hours = int(total_time // 3600) - minutes = int((total_time % 3600) // 60) - seconds = round(total_time % 60, 2) - - if total_time < 60: - formatted_time = f"{seconds} seconds" - elif total_time < 3600: - formatted_time = f"{minutes} minutes {seconds:.2f} seconds" - else: - formatted_time = f"{hours} hours {minutes} minutes {seconds:.2f} seconds" + flight_distance_km = total_distance / 1000 # Convert to kilometers + flight_time_minutes = total_time / 60 # Convert to minutes return { - "total_flight_time": formatted_time, + "total_flight_time": f"{flight_time_minutes:.2f}", "total_flight_time_seconds": round(total_time, 2), + "flight_distance_km": round(flight_distance_km, 2), } diff --git a/src/frontend/src/components/IndividualProject/Tasks/TableSection/index.tsx b/src/frontend/src/components/IndividualProject/Tasks/TableSection/index.tsx index 22c3f20e..156d90a2 100644 --- a/src/frontend/src/components/IndividualProject/Tasks/TableSection/index.tsx +++ b/src/frontend/src/components/IndividualProject/Tasks/TableSection/index.tsx @@ -10,6 +10,14 @@ const tasksDataColumns = [ { header: 'Task Area in km²', accessorKey: 'task_area', + }, + { + header: 'Flight Time in Minutes', + accessorKey: 'flight_time_minutes', + }, + { + header: 'Flight Distance in km', + accessorKey: 'flight_distance_km', } ]; @@ -35,6 +43,8 @@ export default function TableSection({ id: `Task# ${curr?.project_task_index}`, flight_time: curr?.flight_time || '-', task_area: Number(curr?.total_area_sqkm)?.toFixed(3), + flight_time_minutes: Number(curr?.flight_time_minutes)?.toFixed(3), + flight_distance_km: Number(curr?.flight_distance_km)?.toFixed(3), task_id: curr?.id, // status: curr?.state, }, From 46e8a78b2eb83159d4286c68e35e51778ef2a5f4 Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Mon, 30 Dec 2024 13:31:11 +0545 Subject: [PATCH 17/36] fix: import errors in project routes --- src/backend/app/projects/project_routes.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index 28f284d1..d355f644 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -28,7 +28,7 @@ from app.projects import project_schemas, project_deps, project_logic, image_processing from app.db import database from app.models.enums import HTTPStatus, State, FlightMode -from app.s3 import s3_client +from app.s3 import add_file_to_bucket, s3_client from app.config import settings from app.users.user_deps import login_required from app.users.user_schemas import AuthUser @@ -309,7 +309,6 @@ async def generate_presigned_url( user: Annotated[AuthUser, Depends(login_required)], data: project_schemas.PresignedUrlRequest, replace_existing: bool = False, - ): """ Generate a pre-signed URL for uploading an image to S3 Bucket. @@ -368,7 +367,7 @@ async def generate_presigned_url( status_code=HTTPStatus.BAD_REQUEST, detail=f"Failed to delete existing image. {e}", ) - + # Generate a new pre-signed URL for the image upload url = client.get_presigned_url( "PUT", From 90592f360eac313c341abd9b57c4e3d00af38f22 Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Mon, 30 Dec 2024 15:46:56 +0545 Subject: [PATCH 18/36] fix: waypoints & waylines counts --- src/backend/app/projects/project_logic.py | 116 ++++++++++++++++++++- src/backend/app/projects/project_routes.py | 85 ++------------- 2 files changed, 125 insertions(+), 76 deletions(-) diff --git a/src/backend/app/projects/project_logic.py b/src/backend/app/projects/project_logic.py index a632a17b..0d9d4d42 100644 --- a/src/backend/app/projects/project_logic.py +++ b/src/backend/app/projects/project_logic.py @@ -1,4 +1,6 @@ import json +import os +import shutil import uuid from loguru import logger as log from fastapi import HTTPException, UploadFile @@ -33,6 +35,7 @@ add_elevation_from_dem, calculate_parameters, create_placemarks, + terrain_following_waylines, ) from app.models.enums import FlightMode @@ -226,8 +229,6 @@ async def create_tasks_from_geojson( flight_distance_km = calculate_flight_time_from_placemarks(placemarks).get( "flight_distance_km" ) - print(f"Flight time: {flight_time_minutes} minutes") - print(f"Flight distance: {flight_distance_km} km") try: if not polygon["geometry"]: continue @@ -492,3 +493,114 @@ async def update_total_image_uploaded( }, ) return True + + +async def process_waypoints_and_waylines( + side_overlap: float, + front_overlap: float, + altitude_from_ground: float, + gsd_cm_px: float, + meters: float, + project_geojson: UploadFile, + is_terrain_follow: bool, + dem: UploadFile, +): + """ + Processes and returns counts of waypoints and waylines. + """ + # Validate the input GeoJSON file + file_name = os.path.splitext(project_geojson.filename) + file_ext = file_name[1] + allowed_extensions = [".geojson", ".json"] + if file_ext not in allowed_extensions: + raise HTTPException(status_code=400, detail="Provide a valid .geojson file") + + # Generate square boundary GeoJSON + content = project_geojson.file.read() + boundary = geojson.loads(content) + geometry = shape(boundary["features"][0]["geometry"]) + centroid = geometry.centroid + center_lon = centroid.x + center_lat = centroid.y + square_geojson = generate_square_geojson(center_lat, center_lon, meters) + + # Prepare common parameters for waypoint creation + forward_overlap = front_overlap if front_overlap else 70 + side_overlap = side_overlap if side_overlap else 70 + parameters = calculate_parameters( + forward_overlap, + side_overlap, + altitude_from_ground, + gsd_cm_px, + 2, + ) + waypoint_params = { + "project_area": square_geojson, + "agl": altitude_from_ground, + "gsd": gsd_cm_px, + "forward_overlap": forward_overlap, + "side_overlap": side_overlap, + "rotation_angle": 0, + "generate_3d": False, # TODO: For 3d imageries drone_flightplan package needs to be updated. + "take_off_point": None, + } + count_data = {"waypoints": 0, "waylines": 0} + + if is_terrain_follow and dem: + temp_dir = f"/tmp/{uuid.uuid4()}" + dem_path = os.path.join(temp_dir, "dem.tif") + + try: + os.makedirs(temp_dir, exist_ok=True) + # Read DEM content into memory and write to the file + file_content = await dem.read() + with open(dem_path, "wb") as file: + file.write(file_content) + + # Process waypoints with terrain-follow elevation + waypoint_params["mode"] = FlightMode.waypoints + points = waypoints.create_waypoint(**waypoint_params) + + # Add elevation data to waypoints + outfile_with_elevation = os.path.join( + temp_dir, "output_file_with_elevation.geojson" + ) + add_elevation_from_dem(dem_path, points, outfile_with_elevation) + + # Read the updated waypoints with elevation + with open(outfile_with_elevation, "r") as inpointsfile: + points_with_elevation = inpointsfile.read() + count_data["waypoints"] = len( + json.loads(points_with_elevation)["features"] + ) + + # Generate waylines from waypoints with elevation + wayline_placemarks = create_placemarks( + geojson.loads(points_with_elevation), parameters + ) + + placemarks = terrain_following_waylines.waypoints2waylines( + wayline_placemarks, 5 + ) + count_data["waylines"] = len(placemarks["features"]) + + except Exception as e: + log.error(f"Error processing DEM: {e}") + + finally: + # Cleanup temporary files and directory + if os.path.exists(temp_dir): + shutil.rmtree(temp_dir) + return count_data + + else: + # Generate waypoints and waylines + waypoint_params["mode"] = FlightMode.waypoints + points = waypoints.create_waypoint(**waypoint_params) + count_data["waypoints"] = len(json.loads(points)["features"]) + + waypoint_params["mode"] = FlightMode.waylines + lines = waypoints.create_waypoint(**waypoint_params) + count_data["waylines"] = len(json.loads(lines)["features"]) + + return count_data diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index d355f644..d0505f64 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -1,6 +1,5 @@ import json import os -import shutil import uuid from typing import Annotated, Optional from uuid import UUID @@ -27,7 +26,7 @@ from shapely.ops import unary_union from app.projects import project_schemas, project_deps, project_logic, image_processing from app.db import database -from app.models.enums import HTTPStatus, State, FlightMode +from app.models.enums import HTTPStatus, State from app.s3 import add_file_to_bucket, s3_client from app.config import settings from app.users.user_deps import login_required @@ -41,10 +40,6 @@ from app.users import user_schemas from app.jaxa.upload_dem import upload_dem_file from minio.deleteobjects import DeleteObject -from drone_flightplan import ( - waypoints, - add_elevation_from_dem, -) router = APIRouter( prefix=f"{settings.API_PREFIX}/projects", @@ -654,76 +649,18 @@ async def get_project_waypoints_counts( user_data: AuthUser = Depends(login_required), ): """ - Count waypoints within AOI. + Count waypoints and waylines within AOI. """ - # Validating for .geojson File. - file_name = os.path.splitext(project_geojson.filename) - file_ext = file_name[1] - allowed_extensions = [".geojson", ".json"] - if file_ext not in allowed_extensions: - raise HTTPException(status_code=400, detail="Provide a valid .geojson file") - - # read entire file - content = await project_geojson.read() - boundary = geojson.loads(content) - geometry = shape(boundary["features"][0]["geometry"]) - centroid = geometry.centroid - center_lon = centroid.x - center_lat = centroid.y - square_geojson = project_logic.generate_square_geojson( - center_lat, center_lon, meters - ) - generate_3d = ( - False # TODO: For 3d imageries drone_flightplan package needs to be updated. + return await project_logic.process_waypoints_and_waylines( + side_overlap, + front_overlap, + altitude_from_ground, + gsd_cm_px, + meters, + project_geojson, + is_terrain_follow, + dem, ) - forward_overlap = front_overlap if front_overlap else 70 - side_overlap = side_overlap if side_overlap else 70 - - # Common parameters for create_waypoint - waypoint_params = { - "project_area": square_geojson, - "agl": altitude_from_ground, - "gsd": gsd_cm_px, - "forward_overlap": forward_overlap, - "side_overlap": side_overlap, - "rotation_angle": 0, - "generate_3d": generate_3d, - "take_off_point": None, - } - - waypoint_params["mode"] = FlightMode.waypoints - points = waypoints.create_waypoint(**waypoint_params) - count_data = {"waypoints": 0, "waylines": 0} - - # Handle terrain-following logic if a DEM is provided - if is_terrain_follow and dem: - temp_dir = f"/tmp/{uuid.uuid4()}" - try: - os.makedirs(temp_dir, exist_ok=True) - dem_path = os.path.join(temp_dir, "dem.tif") - outfile_with_elevation = os.path.join( - temp_dir, "output_file_with_elevation.geojson" - ) - - with open(dem_path, "wb") as dem_file: - dem_file.write(await dem.read()) - - add_elevation_from_dem(dem_path, waypoints, outfile_with_elevation) - - except Exception as e: - log.error(f"Error processing DEM: {e}") - - finally: - if os.path.exists(temp_dir): - shutil.rmtree(temp_dir) - - count_data["waypoints"] = len(json.loads(points)["features"]) - else: - waypoint_params["mode"] = FlightMode.waylines - lines = waypoints.create_waypoint(**waypoint_params) - count_data["waypoints"] = len(json.loads(points)["features"]) - count_data["waylines"] = len(json.loads(lines)["features"]) - return count_data @router.get( From 8c85e8ade9f513f5d6add8bacf423269fba096d3 Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Mon, 30 Dec 2024 16:45:50 +0545 Subject: [PATCH 19/36] fix: only get unique task id based on task events when all image processing.. --- src/backend/app/projects/project_logic.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/backend/app/projects/project_logic.py b/src/backend/app/projects/project_logic.py index 0d9d4d42..4b88e617 100644 --- a/src/backend/app/projects/project_logic.py +++ b/src/backend/app/projects/project_logic.py @@ -459,13 +459,17 @@ def generate_square_geojson(center_lat, center_lon, side_length_meters): async def get_all_tasks_for_project(project_id, db): - "Get all tasks associated with the project ID that are in state IMAGE_UPLOADED." + """ + Get all unique tasks associated with the project ID + that are in state IMAGE_UPLOADED. + """ async with db.cursor() as cur: query = """ - SELECT t.id + SELECT DISTINCT ON (t.id) t.id FROM tasks t JOIN task_events te ON t.id = te.task_id - WHERE t.project_id = %s AND te.state = 'IMAGE_UPLOADED'; + WHERE t.project_id = %s AND te.state = 'IMAGE_UPLOADED' + ORDER BY t.id, te.created_at DESC; """ await cur.execute(query, (project_id,)) results = await cur.fetchall() From ee51fad3791ca6fd2526514a1a30d5b23ec6fc42 Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Mon, 30 Dec 2024 17:06:07 +0545 Subject: [PATCH 20/36] fix: issues resolved in user task out lists in dashboard --- src/backend/app/tasks/task_schemas.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/app/tasks/task_schemas.py b/src/backend/app/tasks/task_schemas.py index 5d10247a..5e2d2111 100644 --- a/src/backend/app/tasks/task_schemas.py +++ b/src/backend/app/tasks/task_schemas.py @@ -168,7 +168,6 @@ class UserTasksStatsOut(BaseModel): total_area_sqkm: Optional[float] = None flight_time_minutes: Optional[float] = None flight_distance_km: Optional[float] = None - outline: Outline created_at: datetime state: str project_id: uuid.UUID From 27b4df69c062bc92190c382704b6b5ed454edafe Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Mon, 30 Dec 2024 17:07:58 +0545 Subject: [PATCH 21/36] fixup! fix: issues resolved in user task out lists in dashboard --- src/backend/app/tasks/task_routes.py | 2 +- src/backend/app/tasks/task_schemas.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/app/tasks/task_routes.py b/src/backend/app/tasks/task_routes.py index cc37f152..86cd558a 100644 --- a/src/backend/app/tasks/task_routes.py +++ b/src/backend/app/tasks/task_routes.py @@ -47,7 +47,7 @@ async def list_tasks( user_id = user_data.id role = user_data.role log.info(f"Fetching tasks for user {user_id} with role: {role}") - return await task_schemas.UserTasksStatsOut.get_tasks_by_user( + return await task_schemas.UserTasksOut.get_tasks_by_user( db, user_id, role, skip, limit ) diff --git a/src/backend/app/tasks/task_schemas.py b/src/backend/app/tasks/task_schemas.py index 5e2d2111..286fa7b4 100644 --- a/src/backend/app/tasks/task_schemas.py +++ b/src/backend/app/tasks/task_schemas.py @@ -163,7 +163,7 @@ async def all(db: Connection, project_id: uuid.UUID): return combined_tasks -class UserTasksStatsOut(BaseModel): +class UserTasksOut(BaseModel): task_id: uuid.UUID total_area_sqkm: Optional[float] = None flight_time_minutes: Optional[float] = None @@ -201,7 +201,7 @@ def format_url(url): async def get_tasks_by_user( db: Connection, user_id: str, role: str, skip: int = 0, limit: int = 50 ): - async with db.cursor(row_factory=class_row(UserTasksStatsOut)) as cur: + async with db.cursor(row_factory=class_row(UserTasksOut)) as cur: await cur.execute( """ SELECT DISTINCT ON (tasks.id) From 7f6907e2815a22a6a71c566b928667eeae409ef6 Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Mon, 30 Dec 2024 19:16:57 +0545 Subject: [PATCH 22/36] feat: update assests_url in task tables instead of searching in s3 --- src/backend/app/db/db_models.py | 3 + .../app/migrations/versions/b18103ac4ab7_.py | 44 +++++++++++ .../app/migrations/versions/d202ea539f6d_.py | 77 ------------------- src/backend/app/projects/image_processing.py | 8 ++ src/backend/app/projects/project_logic.py | 13 ++-- src/backend/app/projects/project_schemas.py | 16 +++- src/backend/app/s3.py | 23 ++---- src/backend/app/tasks/task_logic.py | 4 +- src/backend/app/tasks/task_schemas.py | 16 +++- 9 files changed, 100 insertions(+), 104 deletions(-) create mode 100644 src/backend/app/migrations/versions/b18103ac4ab7_.py delete mode 100644 src/backend/app/migrations/versions/d202ea539f6d_.py diff --git a/src/backend/app/db/db_models.py b/src/backend/app/db/db_models.py index 8d8f159e..f6c43a21 100644 --- a/src/backend/app/db/db_models.py +++ b/src/backend/app/db/db_models.py @@ -82,6 +82,9 @@ class DbTask(Base): flight_time_minutes = cast(int, Column(Float, nullable=True)) flight_distance_km = cast(float, Column(Float, nullable=True)) total_image_uploaded = cast(int, Column(SmallInteger, nullable=True)) + assets_url = cast( + str, Column(String, nullable=True) + ) # download link for assets of images(orthophoto) class DbProject(Base): diff --git a/src/backend/app/migrations/versions/b18103ac4ab7_.py b/src/backend/app/migrations/versions/b18103ac4ab7_.py new file mode 100644 index 00000000..fc117ba5 --- /dev/null +++ b/src/backend/app/migrations/versions/b18103ac4ab7_.py @@ -0,0 +1,44 @@ +""" + +Revision ID: b18103ac4ab7 +Revises: e23c05f21542 +Create Date: 2024-12-30 11:36:29.762485 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'b18103ac4ab7' +down_revision: Union[str, None] = 'e23c05f21542' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('task_events', 'state', + existing_type=postgresql.ENUM('REQUEST_FOR_MAPPING', 'UNLOCKED_TO_MAP', 'LOCKED_FOR_MAPPING', 'UNLOCKED_TO_VALIDATE', 'LOCKED_FOR_VALIDATION', 'UNLOCKED_DONE', 'UNFLYABLE_TASK', 'IMAGE_UPLOADED', 'IMAGE_PROCESSING_FAILED', 'IMAGE_PROCESSING_STARTED', 'IMAGE_PROCESSING_FINISHED', name='state'), + nullable=False) + op.add_column('tasks', sa.Column('total_area_sqkm', sa.Float(), nullable=True)) + op.add_column('tasks', sa.Column('flight_time_minutes', sa.Float(), nullable=True)) + op.add_column('tasks', sa.Column('flight_distance_km', sa.Float(), nullable=True)) + op.add_column('tasks', sa.Column('total_image_uploaded', sa.SmallInteger(), nullable=True)) + op.add_column('tasks', sa.Column('assets_url', sa.String(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('tasks', 'assets_url') + op.drop_column('tasks', 'total_image_uploaded') + op.drop_column('tasks', 'flight_distance_km') + op.drop_column('tasks', 'flight_time_minutes') + op.drop_column('tasks', 'total_area_sqkm') + op.alter_column('task_events', 'state', + existing_type=postgresql.ENUM('REQUEST_FOR_MAPPING', 'UNLOCKED_TO_MAP', 'LOCKED_FOR_MAPPING', 'UNLOCKED_TO_VALIDATE', 'LOCKED_FOR_VALIDATION', 'UNLOCKED_DONE', 'UNFLYABLE_TASK', 'IMAGE_UPLOADED', 'IMAGE_PROCESSING_FAILED', 'IMAGE_PROCESSING_STARTED', 'IMAGE_PROCESSING_FINISHED', name='state'), + nullable=True) + # ### end Alembic commands ### diff --git a/src/backend/app/migrations/versions/d202ea539f6d_.py b/src/backend/app/migrations/versions/d202ea539f6d_.py deleted file mode 100644 index 9830f62e..00000000 --- a/src/backend/app/migrations/versions/d202ea539f6d_.py +++ /dev/null @@ -1,77 +0,0 @@ -""" - -Revision ID: d202ea539f6d -Revises: e23c05f21542 -Create Date: 2024-12-26 08:11:00.011691 - -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision: str = "d202ea539f6d" -down_revision: Union[str, None] = "e23c05f21542" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.alter_column( - "task_events", - "state", - existing_type=postgresql.ENUM( - "REQUEST_FOR_MAPPING", - "UNLOCKED_TO_MAP", - "LOCKED_FOR_MAPPING", - "UNLOCKED_TO_VALIDATE", - "LOCKED_FOR_VALIDATION", - "UNLOCKED_DONE", - "UNFLYABLE_TASK", - "IMAGE_UPLOADED", - "IMAGE_PROCESSING_FAILED", - "IMAGE_PROCESSING_STARTED", - "IMAGE_PROCESSING_FINISHED", - name="state", - ), - nullable=False, - ) - op.add_column("tasks", sa.Column("total_area_sqkm", sa.Float(), nullable=True)) - op.add_column("tasks", sa.Column("flight_time_minutes", sa.Float(), nullable=True)) - op.add_column("tasks", sa.Column("flight_distance_km", sa.Float(), nullable=True)) - op.add_column( - "tasks", sa.Column("total_image_uploaded", sa.SmallInteger(), nullable=True) - ) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_column("tasks", "total_image_uploaded") - op.drop_column("tasks", "flight_distance_km") - op.drop_column("tasks", "flight_time_minutes") - op.drop_column("tasks", "total_area_sqkm") - op.alter_column( - "task_events", - "state", - existing_type=postgresql.ENUM( - "REQUEST_FOR_MAPPING", - "UNLOCKED_TO_MAP", - "LOCKED_FOR_MAPPING", - "UNLOCKED_TO_VALIDATE", - "LOCKED_FOR_VALIDATION", - "UNLOCKED_DONE", - "UNFLYABLE_TASK", - "IMAGE_UPLOADED", - "IMAGE_PROCESSING_FAILED", - "IMAGE_PROCESSING_STARTED", - "IMAGE_PROCESSING_FINISHED", - name="state", - ), - nullable=True, - ) - # ### end Alembic commands ### diff --git a/src/backend/app/projects/image_processing.py b/src/backend/app/projects/image_processing.py index d9a4ead2..ff5f7ba5 100644 --- a/src/backend/app/projects/image_processing.py +++ b/src/backend/app/projects/image_processing.py @@ -8,6 +8,7 @@ from app.models.enums import State from app.utils import timestamp from app.db import database +from app.projects import project_logic from pyodm import Node from app.s3 import get_file_from_bucket, list_objects_from_bucket, add_file_to_bucket from loguru import logger as log @@ -424,6 +425,13 @@ async def process_assets_from_odm( log.info( f"Task {dtm_task_id} state updated to IMAGE_PROCESSING_FINISHED in the database." ) + s3_path_url = ( + f"dtm-data/projects/{dtm_project_id}/{dtm_task_id}/assets.zip" + ) + # update the task table + await project_logic.update_task_field( + conn, dtm_project_id, dtm_task_id, "assets_url", s3_path_url + ) except Exception as e: log.error(f"Error during processing for project {dtm_project_id}: {e}") diff --git a/src/backend/app/projects/project_logic.py b/src/backend/app/projects/project_logic.py index 4b88e617..f20326ca 100644 --- a/src/backend/app/projects/project_logic.py +++ b/src/backend/app/projects/project_logic.py @@ -1,6 +1,7 @@ import json import os import shutil +from typing import Any import uuid from loguru import logger as log from fastapi import HTTPException, UploadFile @@ -477,21 +478,21 @@ async def get_all_tasks_for_project(project_id, db): return [str(result[0]) for result in results] -async def update_total_image_uploaded( - db: Connection, project_id: uuid.UUID, task_id: uuid.UUID, total_image_count: str +async def update_task_field( + db: Connection, project_id: uuid.UUID, task_id: uuid.UUID, column: Any, value: str ): """ - Update the total_image_uploaded field in the tasks table. + Generic function to update a field(assets_url and total_image_count) in the tasks table. """ async with db.cursor() as cur: await cur.execute( - """ + f""" UPDATE tasks - SET total_image_uploaded = %(total_image_uploaded)s + SET {column} = %(value)s WHERE project_id = %(project_id)s AND id = %(task_id)s; """, { - "total_image_uploaded": total_image_count, + "value": value, "project_id": str(project_id), "task_id": str(task_id), }, diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 69239db3..73aaf67d 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -25,7 +25,7 @@ ) from psycopg.rows import dict_row from app.config import settings -from app.s3 import get_presigned_url +from app.s3 import generate_static_url, get_presigned_url class CentroidOut(BaseModel): @@ -179,6 +179,16 @@ class TaskOut(BaseModel): total_area_sqkm: Optional[float] = None flight_time_minutes: Optional[float] = None flight_distance_km: Optional[float] = None + total_image_uploaded: Optional[int] = None + + @model_validator(mode="after") + def set_assets_url(cls, values): + """Set image_url before rendering the model.""" + assets_url = values.assets_url + if assets_url: + values.assets_url = generate_static_url(settings.S3_BUCKET_NAME, assets_url) + + return values class DbProject(BaseModel): @@ -295,6 +305,8 @@ async def one(db: Connection, project_id: uuid.UUID): t.total_area_sqkm, t.flight_time_minutes, t.flight_distance_km, + t.assets_url, + t.total_image_uploaded, ST_AsGeoJSON(t.outline)::jsonb -> 'coordinates' AS coordinates, ST_AsGeoJSON(t.outline)::jsonb -> 'type' AS type, ST_XMin(ST_Envelope(t.outline)) AS xmin, @@ -323,6 +335,8 @@ async def one(db: Connection, project_id: uuid.UUID): total_area_sqkm, flight_distance_km, flight_time_minutes, + total_image_uploaded, + assets_url, jsonb_build_object( 'type', 'Feature', 'geometry', jsonb_build_object( diff --git a/src/backend/app/s3.py b/src/backend/app/s3.py index 22d83113..afdc3778 100644 --- a/src/backend/app/s3.py +++ b/src/backend/app/s3.py @@ -4,6 +4,7 @@ from io import BytesIO from typing import Any from datetime import timedelta +from urllib.parse import urljoin def s3_client(): @@ -215,19 +216,9 @@ def get_object_metadata(bucket_name: str, object_name: str): return client.stat_object(bucket_name, object_name) -def get_cog_path(bucket_name: str, project_id: str, task_id: str): - """Generate the presigned URL for a COG file in an S3 bucket. - - Args: - bucket_name (str): The name of the S3 bucket. - project_id (str): The unique project identifier. - orthophoto_name (str): The name of the COG file. - - Returns: - str: The presigned URL to access the COG file. - """ - # Construct the S3 path for the COG file - s3_path = f"dtm-data/projects/{project_id}/{task_id}/orthophoto/odm_orthophoto.tif" - - # Get the presigned URL - return get_presigned_url(bucket_name, s3_path) +def generate_static_url(bucket_name: str, s3_path: str): + """Generate a static URL for an S3 object.""" + minio_url, is_secure = is_connection_secure(settings.S3_ENDPOINT) + protocol = "https" if is_secure else "http" + base_url = f"{protocol}://{minio_url}/{bucket_name}/" + return urljoin(base_url, s3_path) diff --git a/src/backend/app/tasks/task_logic.py b/src/backend/app/tasks/task_logic.py index 57215849..d23777da 100644 --- a/src/backend/app/tasks/task_logic.py +++ b/src/backend/app/tasks/task_logic.py @@ -608,8 +608,8 @@ async def handle_event( project_id, task_id ).image_count - await project_logic.update_total_image_uploaded( - db, project_id, task_id, toatl_image_count + await project_logic.update_task_field( + db, project_id, task_id, "total_image_uploaded", toatl_image_count ) return await update_task_state( diff --git a/src/backend/app/tasks/task_schemas.py b/src/backend/app/tasks/task_schemas.py index 286fa7b4..8dc57708 100644 --- a/src/backend/app/tasks/task_schemas.py +++ b/src/backend/app/tasks/task_schemas.py @@ -9,7 +9,7 @@ from psycopg.rows import class_row, dict_row from typing import List, Literal, Optional from pydantic.functional_validators import field_validator -from app.s3 import is_connection_secure +from app.s3 import generate_static_url, is_connection_secure class Geometry(BaseModel): @@ -270,6 +270,8 @@ class TaskDetailsOut(BaseModel): total_area_sqkm: Optional[float] = None flight_time_minutes: Optional[float] = None flight_distance_km: Optional[float] = None + total_image_uploaded: Optional[int] = None + assets_url: Optional[str] = None outline: Outline created_at: datetime updated_at: Optional[datetime] = None @@ -282,6 +284,15 @@ class TaskDetailsOut(BaseModel): gimble_angles_degrees: Optional[int] = None centroid: dict + @model_validator(mode="after") + def set_assets_url(cls, values): + """Set image_url before rendering the model.""" + assets_url = values.assets_url + if assets_url: + values.assets_url = generate_static_url(settings.S3_BUCKET_NAME, assets_url) + + return values + @field_validator("state", mode="after") @classmethod def integer_state_to_string(cls, value: State): @@ -309,7 +320,8 @@ async def get_task_details(db: Connection, task_id: uuid.UUID): tasks.total_area_sqkm, tasks.flight_time_minutes, tasks.flight_distance_km, - + tasks.total_image_uploaded, + tasks.assets_url, -- Construct the outline as a GeoJSON Feature jsonb_build_object( 'type', 'Feature', From b33b4d87d3ff02c908eb860809cf563dbcdb4b92 Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Mon, 30 Dec 2024 19:18:53 +0545 Subject: [PATCH 23/36] fix: run pre-commit for format migartions file --- .../app/migrations/versions/b18103ac4ab7_.py | 71 ++++++++++++++----- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/src/backend/app/migrations/versions/b18103ac4ab7_.py b/src/backend/app/migrations/versions/b18103ac4ab7_.py index fc117ba5..54ddb74c 100644 --- a/src/backend/app/migrations/versions/b18103ac4ab7_.py +++ b/src/backend/app/migrations/versions/b18103ac4ab7_.py @@ -5,6 +5,7 @@ Create Date: 2024-12-30 11:36:29.762485 """ + from typing import Sequence, Union from alembic import op @@ -12,33 +13,67 @@ from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision: str = 'b18103ac4ab7' -down_revision: Union[str, None] = 'e23c05f21542' +revision: str = "b18103ac4ab7" +down_revision: Union[str, None] = "e23c05f21542" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('task_events', 'state', - existing_type=postgresql.ENUM('REQUEST_FOR_MAPPING', 'UNLOCKED_TO_MAP', 'LOCKED_FOR_MAPPING', 'UNLOCKED_TO_VALIDATE', 'LOCKED_FOR_VALIDATION', 'UNLOCKED_DONE', 'UNFLYABLE_TASK', 'IMAGE_UPLOADED', 'IMAGE_PROCESSING_FAILED', 'IMAGE_PROCESSING_STARTED', 'IMAGE_PROCESSING_FINISHED', name='state'), - nullable=False) - op.add_column('tasks', sa.Column('total_area_sqkm', sa.Float(), nullable=True)) - op.add_column('tasks', sa.Column('flight_time_minutes', sa.Float(), nullable=True)) - op.add_column('tasks', sa.Column('flight_distance_km', sa.Float(), nullable=True)) - op.add_column('tasks', sa.Column('total_image_uploaded', sa.SmallInteger(), nullable=True)) - op.add_column('tasks', sa.Column('assets_url', sa.String(), nullable=True)) + op.alter_column( + "task_events", + "state", + existing_type=postgresql.ENUM( + "REQUEST_FOR_MAPPING", + "UNLOCKED_TO_MAP", + "LOCKED_FOR_MAPPING", + "UNLOCKED_TO_VALIDATE", + "LOCKED_FOR_VALIDATION", + "UNLOCKED_DONE", + "UNFLYABLE_TASK", + "IMAGE_UPLOADED", + "IMAGE_PROCESSING_FAILED", + "IMAGE_PROCESSING_STARTED", + "IMAGE_PROCESSING_FINISHED", + name="state", + ), + nullable=False, + ) + op.add_column("tasks", sa.Column("total_area_sqkm", sa.Float(), nullable=True)) + op.add_column("tasks", sa.Column("flight_time_minutes", sa.Float(), nullable=True)) + op.add_column("tasks", sa.Column("flight_distance_km", sa.Float(), nullable=True)) + op.add_column( + "tasks", sa.Column("total_image_uploaded", sa.SmallInteger(), nullable=True) + ) + op.add_column("tasks", sa.Column("assets_url", sa.String(), nullable=True)) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('tasks', 'assets_url') - op.drop_column('tasks', 'total_image_uploaded') - op.drop_column('tasks', 'flight_distance_km') - op.drop_column('tasks', 'flight_time_minutes') - op.drop_column('tasks', 'total_area_sqkm') - op.alter_column('task_events', 'state', - existing_type=postgresql.ENUM('REQUEST_FOR_MAPPING', 'UNLOCKED_TO_MAP', 'LOCKED_FOR_MAPPING', 'UNLOCKED_TO_VALIDATE', 'LOCKED_FOR_VALIDATION', 'UNLOCKED_DONE', 'UNFLYABLE_TASK', 'IMAGE_UPLOADED', 'IMAGE_PROCESSING_FAILED', 'IMAGE_PROCESSING_STARTED', 'IMAGE_PROCESSING_FINISHED', name='state'), - nullable=True) + op.drop_column("tasks", "assets_url") + op.drop_column("tasks", "total_image_uploaded") + op.drop_column("tasks", "flight_distance_km") + op.drop_column("tasks", "flight_time_minutes") + op.drop_column("tasks", "total_area_sqkm") + op.alter_column( + "task_events", + "state", + existing_type=postgresql.ENUM( + "REQUEST_FOR_MAPPING", + "UNLOCKED_TO_MAP", + "LOCKED_FOR_MAPPING", + "UNLOCKED_TO_VALIDATE", + "LOCKED_FOR_VALIDATION", + "UNLOCKED_DONE", + "UNFLYABLE_TASK", + "IMAGE_UPLOADED", + "IMAGE_PROCESSING_FAILED", + "IMAGE_PROCESSING_STARTED", + "IMAGE_PROCESSING_FINISHED", + name="state", + ), + nullable=True, + ) # ### end Alembic commands ### From 651afc86ae46e662f1a0ed1341ddcb24d60d9fd0 Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Tue, 31 Dec 2024 09:42:00 +0545 Subject: [PATCH 24/36] fix: process assests from odm, download issues --- src/backend/app/projects/image_processing.py | 29 +++++++++++++------- src/backend/app/projects/project_logic.py | 2 +- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/backend/app/projects/image_processing.py b/src/backend/app/projects/image_processing.py index ff5f7ba5..03cc7cf0 100644 --- a/src/backend/app/projects/image_processing.py +++ b/src/backend/app/projects/image_processing.py @@ -363,13 +363,19 @@ async def process_assets_from_odm( """ log.info(f"Starting processing for project {dtm_project_id}") node = Node.from_url(node_odm_url) - output_file_path = f"/tmp/{dtm_project_id}" + output_file_path = f"/tmp/{uuid.uuid4()}" try: + os.makedirs(output_file_path, exist_ok=True) task = node.get_task(odm_task_id) - log.info(f"Downloading results for task {dtm_project_id} to {output_file_path}") + log.info(f"Downloading results for task {odm_task_id} to {output_file_path}") assets_path = task.download_zip(output_file_path) + if not os.path.exists(assets_path): + log.error(f"Downloaded file not found: {assets_path}") + raise + log.info(f"Successfully downloaded ZIP to {assets_path}") + s3_path = f"dtm-data/projects/{dtm_project_id}/{dtm_task_id if dtm_task_id else ''}/assets.zip".strip( "/" ) @@ -395,14 +401,16 @@ async def process_assets_from_odm( add_file_to_bucket(settings.S3_BUCKET_NAME, orthophoto_path, s3_ortho_path) images_json_path = os.path.join(output_file_path, "images.json") - s3_images_json_path = f"dtm-data/projects/{dtm_project_id}/{dtm_task_id if dtm_task_id else ''}/images.json".strip( - "/" - ) - - log.info(f"Uploading images.json to S3 path: {s3_images_json_path}") - add_file_to_bucket( - settings.S3_BUCKET_NAME, images_json_path, s3_images_json_path - ) + if os.path.exists(images_json_path): + s3_images_json_path = f"dtm-data/projects/{dtm_project_id}/{dtm_task_id if dtm_task_id else ''}/images.json".strip( + "/" + ) + log.info(f"Uploading images.json to S3 path: {s3_images_json_path}") + add_file_to_bucket( + settings.S3_BUCKET_NAME, images_json_path, s3_images_json_path + ) + else: + log.warning(f"images.json not found in {output_file_path}") log.info(f"Processing complete for project {dtm_project_id}") @@ -425,6 +433,7 @@ async def process_assets_from_odm( log.info( f"Task {dtm_task_id} state updated to IMAGE_PROCESSING_FINISHED in the database." ) + s3_path_url = ( f"dtm-data/projects/{dtm_project_id}/{dtm_task_id}/assets.zip" ) diff --git a/src/backend/app/projects/project_logic.py b/src/backend/app/projects/project_logic.py index f20326ca..4eac3969 100644 --- a/src/backend/app/projects/project_logic.py +++ b/src/backend/app/projects/project_logic.py @@ -51,7 +51,7 @@ async def get_centroids(db: Connection): p.name, ST_AsGeoJSON(p.centroid)::jsonb AS centroid, COUNT(t.id) AS total_task_count, - COUNT(CASE WHEN te.state IN ('LOCKED_FOR_MAPPING', 'REQUEST_FOR_MAPPING', 'IMAGE_UPLOADED', 'UNFLYABLE_TASK') THEN 1 END) AS ongoing_task_count, + COUNT(CASE WHEN te.state IN ('LOCKED_FOR_MAPPING', 'REQUEST_FOR_MAPPING', 'IMAGE_UPLOADED', 'UNFLYABLE_TASK', 'IMAGE_PROCESSING_STARTED') THEN 1 END) AS ongoing_task_count, COUNT(CASE WHEN te.state = 'IMAGE_PROCESSING_FINISHED' THEN 1 END) AS completed_task_count FROM projects p From 8f4289a6acefcf36e46a051d9fa5f23bcfdfeeca Mon Sep 17 00:00:00 2001 From: Sujit Date: Tue, 31 Dec 2024 10:27:45 +0545 Subject: [PATCH 25/36] feat: add dem file on task split api payload --- .../FormContents/GenerateTask/index.tsx | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/frontend/src/components/CreateProject/FormContents/GenerateTask/index.tsx b/src/frontend/src/components/CreateProject/FormContents/GenerateTask/index.tsx index dffe4754..07adf16d 100644 --- a/src/frontend/src/components/CreateProject/FormContents/GenerateTask/index.tsx +++ b/src/frontend/src/components/CreateProject/FormContents/GenerateTask/index.tsx @@ -18,6 +18,10 @@ import MapSection from './MapSection'; export default function GenerateTask({ formProps }: { formProps: any }) { const dispatch = useTypedDispatch(); const [error, setError] = useState(''); + const isTerrainFollow = useTypedSelector( + state => state.createproject.isTerrainFollow, + ); + const demType = useTypedSelector(state => state.createproject.demType); const { register, watch } = formProps; const { @@ -27,6 +31,7 @@ export default function GenerateTask({ formProps }: { formProps: any }) { gsd_cm_px, outline, task_split_dimension, + dem: demFile, } = watch(); const dimension = watch('task_split_dimension'); @@ -53,9 +58,10 @@ export default function GenerateTask({ formProps }: { formProps: any }) { isLoading: projectWaypointCountIsLoading, } = useMutation({ mutationFn: (projectGeoJsonPayload: Record) => { - const { project_geojson, ...params } = projectGeoJsonPayload; + const { project_geojson, dem, ...params } = projectGeoJsonPayload; return getProjectWayPoints(params, { project_geojson, + dem, }); }, }); @@ -71,6 +77,8 @@ export default function GenerateTask({ formProps }: { formProps: any }) { }, }); + // console.log(dem, isTerrainFollow, demType, 'types'); + return (
@@ -115,6 +123,11 @@ export default function GenerateTask({ formProps }: { formProps: any }) { gsd_cm_px: gsd_cm_px || 0, meters: task_split_dimension, project_geojson: convertGeojsonToFile(outline), + is_terrain_follow: isTerrainFollow, + dem: + isTerrainFollow && demType === 'manual' + ? demFile[0]?.file + : null, }; mutateProjectWayPoints(projectWayPointsPayload); return mutate(payload); From fd97e15b3ce4cf920d7687a0bfb35c38fe651b20 Mon Sep 17 00:00:00 2001 From: Sujit Date: Tue, 31 Dec 2024 10:37:33 +0545 Subject: [PATCH 26/36] fix: dem data upload section is on view althoiugh the terrian follow option is false --- .../CreateProject/FormContents/KeyParameters/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/src/components/CreateProject/FormContents/KeyParameters/index.tsx b/src/frontend/src/components/CreateProject/FormContents/KeyParameters/index.tsx index 5f0c4a7e..1d9df04d 100644 --- a/src/frontend/src/components/CreateProject/FormContents/KeyParameters/index.tsx +++ b/src/frontend/src/components/CreateProject/FormContents/KeyParameters/index.tsx @@ -384,7 +384,7 @@ const KeyParameters = ({ formProps }: { formProps: UseFormPropsType }) => { )} - {demType === 'manual' && ( + {demType === 'manual' && isTerrainFollow && ( Date: Tue, 31 Dec 2024 10:53:44 +0545 Subject: [PATCH 27/36] fix: projection creation fail if no fly is [] remove no flyzone key if np fly zone data is not available --- .../src/components/CreateProject/CreateprojectLayout/index.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/frontend/src/components/CreateProject/CreateprojectLayout/index.tsx b/src/frontend/src/components/CreateProject/CreateprojectLayout/index.tsx index 21b84133..6d4eba59 100644 --- a/src/frontend/src/components/CreateProject/CreateprojectLayout/index.tsx +++ b/src/frontend/src/components/CreateProject/CreateprojectLayout/index.tsx @@ -293,8 +293,7 @@ const CreateprojectLayout = () => { delete refactoredData?.side_spacing; // remove key - if (isNoflyzonePresent === 'no') - delete refactoredData?.outline_no_fly_zones; + if (isNoflyzonePresent === 'no') delete refactoredData?.no_fly_zones; delete refactoredData?.dem; if (measurementType === 'gsd') delete refactoredData?.altitude_from_ground; else delete refactoredData?.gsd_cm_px; From ddf79f6cc86dde6e59ee7735e17523858e51f476 Mon Sep 17 00:00:00 2001 From: Sujit Date: Tue, 31 Dec 2024 11:49:06 +0545 Subject: [PATCH 28/36] feat: remove unused api service `getAllAssetsUrl` --- src/frontend/src/api/projects.ts | 24 ++++++++++++------------ src/frontend/src/services/project.ts | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/frontend/src/api/projects.ts b/src/frontend/src/api/projects.ts index 12fced39..28b15d38 100644 --- a/src/frontend/src/api/projects.ts +++ b/src/frontend/src/api/projects.ts @@ -5,7 +5,7 @@ import { getProjectDetail, getProjectCentroid, } from '@Services/createproject'; -import { getAllAssetsUrl, getTaskStates } from '@Services/project'; +import { getTaskStates } from '@Services/project'; import { getUserProfileInfo } from '@Services/common'; export const useGetProjectsListQuery = ( @@ -66,17 +66,17 @@ export const useGetUserDetailsQuery = ( }); }; -export const useGetAllAssetsUrlQuery = ( - projectId: string, - queryOptions?: Partial, -) => { - return useQuery({ - queryKey: ['all-assets-url'], - queryFn: () => getAllAssetsUrl(projectId), - select: (data: any) => data.data, - ...queryOptions, - }); -}; +// export const useGetAllAssetsUrlQuery = ( +// projectId: string, +// queryOptions?: Partial, +// ) => { +// return useQuery({ +// queryKey: ['all-assets-url'], +// queryFn: () => getAllAssetsUrl(projectId), +// select: (data: any) => data.data, +// ...queryOptions, +// }); +// }; export const useGetProjectCentroidQuery = ( queryOptions?: Partial, diff --git a/src/frontend/src/services/project.ts b/src/frontend/src/services/project.ts index c5db4348..2b4ffc45 100644 --- a/src/frontend/src/services/project.ts +++ b/src/frontend/src/services/project.ts @@ -13,5 +13,5 @@ export const postTaskStatus = (payload: Record) => { export const getRequestedTasks = () => authenticated(api).get('/tasks/requested_tasks/pending'); -export const getAllAssetsUrl = (projectId: string) => - authenticated(api).get(`/projects/assets/${projectId}/`); +// export const getAllAssetsUrl = (projectId: string) => +// authenticated(api).get(`/projects/assets/${projectId}/`); From e8a51d26871341b0c5428683e1bdedcce1217fd5 Mon Sep 17 00:00:00 2001 From: Sujit Date: Tue, 31 Dec 2024 11:50:52 +0545 Subject: [PATCH 29/36] feat: add remove project assets api fetch and display data from project description api --- .../Contributions/TableSection/index.tsx | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/src/frontend/src/components/IndividualProject/Contributions/TableSection/index.tsx b/src/frontend/src/components/IndividualProject/Contributions/TableSection/index.tsx index 9b6656b3..6164f599 100644 --- a/src/frontend/src/components/IndividualProject/Contributions/TableSection/index.tsx +++ b/src/frontend/src/components/IndividualProject/Contributions/TableSection/index.tsx @@ -1,4 +1,3 @@ -import { useGetAllAssetsUrlQuery } from '@Api/projects'; import DataTable from '@Components/common/DataTable'; import Icon from '@Components/common/Icon'; import { toggleModal } from '@Store/actions/common'; @@ -7,7 +6,6 @@ import { useTypedSelector } from '@Store/hooks'; import { formatString } from '@Utils/index'; import { useMemo } from 'react'; import { useDispatch } from 'react-redux'; -import { useParams } from 'react-router-dom'; import { toast } from 'react-toastify'; const contributionsDataColumns = [ @@ -93,37 +91,27 @@ export default function TableSection({ isFetching, handleTableRowClick, }: ITableSectionProps) { - const { id } = useParams(); const tasksData = useTypedSelector(state => state.project.tasksData); - const { data: allUrls, isFetching: isUrlFetching } = useGetAllAssetsUrlQuery( - id as string, - ); - - const getTasksAssets = (taskID: string, assetsList: any[]) => { - if (!assetsList || !taskID) return null; - return assetsList.find((assets: any) => assets?.task_id === taskID); - }; - const taskDataForTable = useMemo(() => { - if (!tasksData || isUrlFetching) return []; + if (!tasksData) return []; return tasksData?.reduce((acc: any, curr: any) => { if (!curr?.state || curr?.state === 'UNLOCKED_TO_MAP') return acc; - const selectedAssetsDetails = getTasksAssets(curr?.id, allUrls as any[]); + return [ ...acc, { user: curr?.name || '-', task_mapped: `Task# ${curr?.project_task_index}`, task_state: formatString(curr?.state), - assets_url: selectedAssetsDetails?.assets_url, - image_count: selectedAssetsDetails?.image_count, + assets_url: curr?.assets_url, + image_count: curr?.total_image_uploaded, task_id: curr?.id, outline: curr?.outline, }, ]; }, []); - }, [tasksData, allUrls, isUrlFetching]); + }, [tasksData]); return ( []} withPagination={false} - loading={isFetching || isUrlFetching} + loading={isFetching} handleTableRowClick={handleTableRowClick} /> ); From fe79a24faa3c592bc5e501032e3d4d72d0e486eb Mon Sep 17 00:00:00 2001 From: Sujit Date: Tue, 31 Dec 2024 14:38:25 +0545 Subject: [PATCH 30/36] feat: add action and slice for storing assets information of task --- src/frontend/src/store/actions/droneOperatorTask.ts | 1 + src/frontend/src/store/slices/droneOperartorTask.ts | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/src/frontend/src/store/actions/droneOperatorTask.ts b/src/frontend/src/store/actions/droneOperatorTask.ts index 9b71d046..38efe121 100644 --- a/src/frontend/src/store/actions/droneOperatorTask.ts +++ b/src/frontend/src/store/actions/droneOperatorTask.ts @@ -17,4 +17,5 @@ export const { setFilesExifData, resetFilesExifData, setWaypointMode, + setTaskAssetsInformation, } = droneOperatorTaskSlice.actions; diff --git a/src/frontend/src/store/slices/droneOperartorTask.ts b/src/frontend/src/store/slices/droneOperartorTask.ts index 6f5319bb..23ce64e3 100644 --- a/src/frontend/src/store/slices/droneOperartorTask.ts +++ b/src/frontend/src/store/slices/droneOperartorTask.ts @@ -19,6 +19,7 @@ export interface IDroneOperatorTaskState { selectedTaskDetailToViewOrthophoto: any; filesExifData: IFilesExifData[]; waypointMode: 'waypoints' | 'waylines'; + taskAssetsInformation: Record; } const initialState: IDroneOperatorTaskState = { @@ -34,6 +35,11 @@ const initialState: IDroneOperatorTaskState = { selectedTaskDetailToViewOrthophoto: null, filesExifData: [], waypointMode: 'waypoints', + taskAssetsInformation: { + total_image_uploaded: 0, + assets_url: '', + state: '', + }, }; export const droneOperatorTaskSlice = createSlice({ @@ -95,6 +101,9 @@ export const droneOperatorTaskSlice = createSlice({ setWaypointMode: (state, action) => { state.waypointMode = action.payload; }, + setTaskAssetsInformation: (state, action) => { + state.taskAssetsInformation = action.payload; + }, }, }); From c27eafa4f4c8402a673870cc7d5b66f4031f3f1b Mon Sep 17 00:00:00 2001 From: Sujit Date: Tue, 31 Dec 2024 14:42:40 +0545 Subject: [PATCH 31/36] feat(task-description-map-section): remove extra call for task information and use data from redux state --- .../DroneOperatorTask/MapSection/index.tsx | 57 +++++++++---------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/src/frontend/src/components/DroneOperatorTask/MapSection/index.tsx b/src/frontend/src/components/DroneOperatorTask/MapSection/index.tsx index 7b728d13..04c51204 100644 --- a/src/frontend/src/components/DroneOperatorTask/MapSection/index.tsx +++ b/src/frontend/src/components/DroneOperatorTask/MapSection/index.tsx @@ -4,7 +4,7 @@ /* eslint-disable react/no-array-index-key */ import { useGetIndividualTaskQuery, - useGetTaskAssetsInfo, + // useGetTaskAssetsInfo, useGetTaskWaypointQuery, } from '@Api/tasks'; import marker from '@Assets/images/marker.png'; @@ -42,7 +42,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useDispatch } from 'react-redux'; import { useParams } from 'react-router-dom'; import ToolTip from '@Components/RadixComponents/ToolTip'; -import Skeleton from '@Components/RadixComponents/Skeleton'; +// import Skeleton from '@Components/RadixComponents/Skeleton'; import rotateGeoJSON from '@Utils/rotateGeojsonData'; import COGOrthophotoViewer from '@Components/common/MapLibreComponents/COGOrthophotoViewer'; import { toast } from 'react-toastify'; @@ -89,6 +89,9 @@ const MapSection = ({ className }: { className?: string }) => { const waypointMode = useTypedSelector( state => state.droneOperatorTask.waypointMode, ); + const taskAssetsInformation = useTypedSelector( + state => state.droneOperatorTask.taskAssetsInformation, + ); function setVisibilityOfLayers(layerIds: string[], visibility: string) { layerIds.forEach(layerId => { @@ -359,13 +362,13 @@ const MapSection = ({ className }: { className?: string }) => { }); }; - const { - data: taskAssetsInformation, - isFetching: taskAssetsInfoLoading, - }: Record = useGetTaskAssetsInfo( - projectId as string, - taskId as string, - ); + // const { + // data: taskAssetsInformation, + // isFetching: taskAssetsInfoLoading, + // }: Record = useGetTaskAssetsInfo( + // projectId as string, + // taskId as string, + // ); useEffect(() => { setTaskWayPoints(taskWayPointsData); @@ -714,26 +717,22 @@ const MapSection = ({ className }: { className?: string }) => {
- {taskAssetsInfoLoading ? ( - - ) : ( - taskAssetsInformation?.assets_url && ( -
- -
- ) + {taskAssetsInformation?.assets_url && ( +
+ +
)} {!isRotationEnabled && (
From 5ce0da9005b8daa689231a71ea5660c70f424cd1 Mon Sep 17 00:00:00 2001 From: Sujit Date: Tue, 31 Dec 2024 14:48:21 +0545 Subject: [PATCH 32/36] feat: remove task-assets-information api and display data from task description api store asests info on redux state on api call success and remove on component unmount update keys as per data information --- .../DescriptionBox/index.tsx | 75 +++++++++---------- .../DroneOperatorTask/MapSection/index.tsx | 8 ++ 2 files changed, 44 insertions(+), 39 deletions(-) diff --git a/src/frontend/src/components/DroneOperatorTask/DescriptionSection/DescriptionBox/index.tsx b/src/frontend/src/components/DroneOperatorTask/DescriptionSection/DescriptionBox/index.tsx index bd6328c1..1dd33da0 100644 --- a/src/frontend/src/components/DroneOperatorTask/DescriptionSection/DescriptionBox/index.tsx +++ b/src/frontend/src/components/DroneOperatorTask/DescriptionSection/DescriptionBox/index.tsx @@ -1,12 +1,8 @@ -import { useEffect, useState } from 'react'; +import { useEffect, useMemo, useState } from 'react'; import { useParams } from 'react-router-dom'; import { useDispatch } from 'react-redux'; import { toast } from 'react-toastify'; -import { - useGetIndividualTaskQuery, - useGetTaskAssetsInfo, - useGetTaskWaypointQuery, -} from '@Api/tasks'; +import { useGetIndividualTaskQuery, useGetTaskWaypointQuery } from '@Api/tasks'; import { useMutation, useQueryClient } from '@tanstack/react-query'; import { postProcessImagery } from '@Services/tasks'; import { formatString } from '@Utils/index'; @@ -16,12 +12,12 @@ import SwitchTab from '@Components/common/SwitchTab'; import { resetFilesExifData, setSelectedTaskDetailToViewOrthophoto, + setTaskAssetsInformation, setUploadedImagesType, } from '@Store/actions/droneOperatorTask'; import { useTypedSelector } from '@Store/hooks'; // import { toggleModal } from '@Store/actions/common'; import { postTaskStatus } from '@Services/project'; -import Skeleton from '@Components/RadixComponents/Skeleton'; import DescriptionBoxComponent from './DescriptionComponent'; import QuestionBox from '../QuestionBox'; import UploadsInformation from '../UploadsInformation'; @@ -49,13 +45,6 @@ const DescriptionBox = () => { }, }, ); - const { - data: taskAssetsInformation, - isFetching: taskAssetsInfoLoading, - }: Record = useGetTaskAssetsInfo( - projectId as string, - taskId as string, - ); const { mutate: updateStatus, isLoading: statusUpdating } = useMutation< any, @@ -93,29 +82,27 @@ const DescriptionBox = () => { }, }); - const { data: flightTimeData }: any = useGetTaskWaypointQuery( - projectId as string, - taskId as string, - waypointMode as string, - { - select: ({ data }: any) => data.flight_data, - }, - ); - const { data: taskDescription }: Record = useGetIndividualTaskQuery(taskId as string, { - enabled: !!taskWayPoints, + // enabled: !!taskWayPoints, select: (data: any) => { const { data: taskData } = data; dispatch( - dispatch( - setSelectedTaskDetailToViewOrthophoto({ - outline: taskData?.outline, - }), - ), + setSelectedTaskDetailToViewOrthophoto({ + outline: taskData?.outline, + }), ); + // dispatch( + // setTaskAssetsInformation({ + // taskAssetsInformation: { + // total_image_uploaded: taskData?.total_image_uploaded || 0, + // assets_url: taskData?.assets_url, + // state: taskData?.state, + // }, + // }), + return [ { id: 1, @@ -145,7 +132,9 @@ const DescriptionBox = () => { }, { name: 'Est. flight time', - value: flightTimeData?.total_flight_time || null, + value: taskData?.flight_time_minutes + ? `${Number(taskData?.flight_time_minutes)?.toFixed(3)} minutes` + : null, }, ], }, @@ -184,17 +173,27 @@ const DescriptionBox = () => { }, ], }, + { + total_image_uploaded: taskData?.total_image_uploaded || 0, + assets_url: taskData?.assets_url, + state: taskData?.state, + }, ]; }, }); + const taskAssetsInformation = useMemo(() => { + if (!taskDescription) return {}; + dispatch(setTaskAssetsInformation(taskDescription?.[2])); + return taskDescription?.[2]; + }, [taskDescription, dispatch]); + const handleDownloadResult = () => { if (!taskAssetsInformation?.assets_url) return; - try { const link = document.createElement('a'); link.href = taskAssetsInformation?.assets_url; - link.download = 'assets.zip'; + link.download = `${projectId}-${taskId}.tif`; document.body.appendChild(link); link.click(); link.remove(); @@ -214,21 +213,21 @@ const DescriptionBox = () => { /> ))}
- {taskAssetsInformation?.image_count === 0 && ( + + {taskAssetsInformation?.total_image_uploaded === 0 && ( )} - - {taskAssetsInformation?.image_count > 0 && ( + {taskAssetsInformation?.total_image_uploaded > 0 && (
{ ]} /> - {taskAssetsInfoLoading && } - {taskAssetsInformation?.assets_url && (
{/*