feat(server): refresh face detection (#12335)

* refresh faces handle non-ml faces * fix metadata face handling * updated tests * added todo comment
immich-app · Oct 4, 2024 · 2c87683 · 2c87683
1 parent 9edc9d6
commit 2c87683
Show file tree

Hide file tree

Showing 21 changed files with 408 additions and 151 deletions.
diff --git a/mobile/openapi/lib/model/asset_job_name.dart b/mobile/openapi/lib/model/asset_job_name.dart
diff --git a/mobile/openapi/lib/model/job_command_dto.dart b/mobile/openapi/lib/model/job_command_dto.dart
diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json
@@ -8215,8 +8215,9 @@
       },
       "AssetJobName": {
         "enum": [
-          "regenerate-thumbnail",
+          "refresh-faces",
           "refresh-metadata",
+          "regenerate-thumbnail",
           "transcode-video"
         ],
         "type": "string"
@@ -9277,8 +9278,7 @@
           }
         },
         "required": [
-          "command",
-          "force"
+          "command"
         ],
         "type": "object"
       },

diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts
@@ -554,7 +554,7 @@ export type JobCreateDto = {
 };
 export type JobCommandDto = {
     command: JobCommand;
-    force: boolean;
+    force?: boolean;
 };
 export type LibraryResponseDto = {
     assetCount: number;
@@ -3426,8 +3426,9 @@ export enum Reason {
     UnsupportedFormat = "unsupported-format"
 }
 export enum AssetJobName {
-    RegenerateThumbnail = "regenerate-thumbnail",
+    RefreshFaces = "refresh-faces",
     RefreshMetadata = "refresh-metadata",
+    RegenerateThumbnail = "regenerate-thumbnail",
     TranscodeVideo = "transcode-video"
 }
 export enum AssetMediaSize {

diff --git a/server/src/dtos/asset.dto.ts b/server/src/dtos/asset.dto.ts
@@ -92,8 +92,9 @@ export class AssetIdsDto {
 }
 
 export enum AssetJobName {
-  REGENERATE_THUMBNAIL = 'regenerate-thumbnail',
+  REFRESH_FACES = 'refresh-faces',
   REFRESH_METADATA = 'refresh-metadata',
+  REGENERATE_THUMBNAIL = 'regenerate-thumbnail',
   TRANSCODE_VIDEO = 'transcode-video',
 }
 

diff --git a/server/src/dtos/job.dto.ts b/server/src/dtos/job.dto.ts
@@ -18,7 +18,7 @@ export class JobCommandDto {
   command!: JobCommand;
 
   @ValidateBoolean({ optional: true })
-  force!: boolean;
+  force?: boolean; // TODO: this uses undefined as a third state, which should be refactored to be more explicit
 }
 
 export class JobCreateDto {

diff --git a/server/src/interfaces/person.interface.ts b/server/src/interfaces/person.interface.ts
@@ -1,5 +1,6 @@
 import { AssetFaceEntity } from 'src/entities/asset-face.entity';
 import { AssetEntity } from 'src/entities/asset.entity';
+import { FaceSearchEntity } from 'src/entities/face-search.entity';
 import { PersonEntity } from 'src/entities/person.entity';
 import { SourceType } from 'src/enum';
 import { Paginated, PaginationOptions } from 'src/utils/pagination';
@@ -63,7 +64,11 @@ export interface IPersonRepository {
   delete(entities: PersonEntity[]): Promise<void>;
   deleteAll(): Promise<void>;
   deleteFaces(options: DeleteFacesOptions): Promise<void>;
-  replaceFaces(assetId: string, entities: Partial<AssetFaceEntity>[], sourceType?: string): Promise<string[]>;
+  refreshFaces(
+    facesToAdd: Partial<AssetFaceEntity>[],
+    faceIdsToRemove: string[],
+    embeddingsToAdd?: FaceSearchEntity[],
+  ): Promise<void>;
   getAllFaces(pagination: PaginationOptions, options?: FindManyOptions<AssetFaceEntity>): Paginated<AssetFaceEntity>;
   getFaceById(id: string): Promise<AssetFaceEntity>;
   getFaceByIdWithAssets(

diff --git a/server/src/repositories/person.repository.ts b/server/src/repositories/person.repository.ts
@@ -5,6 +5,7 @@ import { ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
 import { AssetFaceEntity } from 'src/entities/asset-face.entity';
 import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
 import { AssetEntity } from 'src/entities/asset.entity';
+import { FaceSearchEntity } from 'src/entities/face-search.entity';
 import { PersonEntity } from 'src/entities/person.entity';
 import { PaginationMode, SourceType } from 'src/enum';
 import {
@@ -31,6 +32,7 @@ export class PersonRepository implements IPersonRepository {
     @InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
     @InjectRepository(PersonEntity) private personRepository: Repository<PersonEntity>,
     @InjectRepository(AssetFaceEntity) private assetFaceRepository: Repository<AssetFaceEntity>,
+    @InjectRepository(FaceSearchEntity) private faceSearchRepository: Repository<FaceSearchEntity>,
     @InjectRepository(AssetJobStatusEntity) private jobStatusRepository: Repository<AssetJobStatusEntity>,
   ) {}
 
@@ -296,12 +298,31 @@ export class PersonRepository implements IPersonRepository {
     return res.map((row) => row.id);
   }
 
-  async replaceFaces(assetId: string, entities: AssetFaceEntity[], sourceType: string): Promise<string[]> {
-    return this.dataSource.transaction(async (manager) => {
-      await manager.delete(AssetFaceEntity, { assetId, sourceType });
-      const assetFaces = await manager.save(AssetFaceEntity, entities);
-      return assetFaces.map(({ id }) => id);
-    });
+  async refreshFaces(
+    facesToAdd: Partial<AssetFaceEntity>[],
+    faceIdsToRemove: string[],
+    embeddingsToAdd?: FaceSearchEntity[],
+  ): Promise<void> {
+    const query = this.faceSearchRepository.createQueryBuilder().select('1');
+    if (facesToAdd.length > 0) {
+      const insertCte = this.assetFaceRepository.createQueryBuilder().insert().values(facesToAdd);
+      query.addCommonTableExpression(insertCte, 'added');
+    }
+
+    if (faceIdsToRemove.length > 0) {
+      const deleteCte = this.assetFaceRepository
+        .createQueryBuilder()
+        .delete()
+        .where('id = any(:faceIdsToRemove)', { faceIdsToRemove });
+      query.addCommonTableExpression(deleteCte, 'deleted');
+    }
+
+    if (embeddingsToAdd?.length) {
+      const embeddingCte = this.faceSearchRepository.createQueryBuilder().insert().values(embeddingsToAdd).orIgnore();
+      query.addCommonTableExpression(embeddingCte, 'embeddings');
+    }
+
+    await query.execute();
   }
 
   async update(person: Partial<PersonEntity>): Promise<PersonEntity> {

diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.ts
@@ -92,9 +92,9 @@ export class AssetService extends BaseService {
       id,
       {
         exifInfo: true,
-        tags: true,
         sharedLinks: true,
         smartInfo: true,
+        tags: true,
         owner: true,
         faces: {
           person: true,
@@ -290,6 +290,11 @@ export class AssetService extends BaseService {
 
     for (const id of dto.assetIds) {
       switch (dto.name) {
+        case AssetJobName.REFRESH_FACES: {
+          jobs.push({ name: JobName.FACE_DETECTION, data: { id } });
+          break;
+        }
+
         case AssetJobName.REFRESH_METADATA: {
           jobs.push({ name: JobName.METADATA_EXTRACTION, data: { id } });
           break;