From d4f80bc30bae44ebd3e747046e523933dd8f766a Mon Sep 17 00:00:00 2001 From: brimoor Date: Fri, 29 Nov 2024 17:22:25 -0600 Subject: [PATCH] add compute_near_duplicates() method --- fiftyone/brain/__init__.py | 153 +++++++++++++++++- fiftyone/brain/internal/core/duplicates.py | 74 +++++++++ fiftyone/brain/internal/core/elasticsearch.py | 18 +-- fiftyone/brain/internal/core/lancedb.py | 19 +-- fiftyone/brain/internal/core/leaky_splits.py | 10 +- fiftyone/brain/internal/core/milvus.py | 20 +-- fiftyone/brain/internal/core/mongodb.py | 29 +--- fiftyone/brain/internal/core/pinecone.py | 20 +-- fiftyone/brain/internal/core/qdrant.py | 20 +-- fiftyone/brain/internal/core/redis.py | 20 +-- .../brain/internal/core/representativeness.py | 2 +- fiftyone/brain/internal/core/sklearn.py | 26 +-- fiftyone/brain/internal/core/uniqueness.py | 2 +- fiftyone/brain/similarity.py | 44 ++++- 14 files changed, 288 insertions(+), 169 deletions(-) diff --git a/fiftyone/brain/__init__.py b/fiftyone/brain/__init__.py index ccb420cf..909f65ff 100644 --- a/fiftyone/brain/__init__.py +++ b/fiftyone/brain/__init__.py @@ -539,6 +539,7 @@ def compute_visualization( def compute_similarity( samples, patches_field=None, + roi_field=None, embeddings=None, brain_key=None, model=None, @@ -592,6 +593,11 @@ def compute_similarity( :class:`fiftyone.core.labels.Detections`, :class:`fiftyone.core.labels.Polyline`, or :class:`fiftyone.core.labels.Polylines` + roi_field (None): an optional :class:`fiftyone.core.labels.Detection`, + :class:`fiftyone.core.labels.Detections`, + :class:`fiftyone.core.labels.Polyline`, or + :class:`fiftyone.core.labels.Polylines` field defining a region of + interest within each image to use to compute embeddings embeddings (None): embeddings to feed the index. This argument's behavior depends on whether a ``model`` is provided, as described below. @@ -600,8 +606,9 @@ def compute_similarity( embeddings to use: - a ``num_samples x num_dims`` array of embeddings - - if ``patches_field`` is specified, a dict mapping sample IDs - to ``num_patches x num_dims`` arrays of patch embeddings + - if ``patches_field``/``roi_field`` is specified, a dict + mapping sample IDs to ``num_patches x num_dims`` arrays of + patch embeddings - the name of a dataset field from which to load embeddings - ``None``: use the default model to compute embeddings - ``False``: **do not** compute embeddings right now @@ -614,7 +621,7 @@ def compute_similarity( In either case, when working with patch embeddings, you can provide either the fully-qualified path to the patch embeddings or just the - name of the label attribute in ``patches_field`` + name of the label attribute in ``patches_field``/``roi_field`` brain_key (None): a brain key under which to store the results of this method model (None): a :class:`fiftyone.core.models.Model` or the name of a @@ -626,14 +633,14 @@ def compute_similarity( to the model's ``Config`` when a model name is provided force_square (False): whether to minimally manipulate the patch bounding boxes into squares prior to extraction. Only applicable - when a ``model`` and ``patches_field`` are specified + when a ``model`` and ``patches_field``/``roi_field`` are specified alpha (None): an optional expansion/contraction to apply to the patches before extracting them, in ``[-1, inf)``. If provided, the length and width of the box are expanded (or contracted, when ``alpha < 0``) by ``(100 * alpha)%``. For example, set ``alpha = 0.1`` to expand the boxes by 10%, and set ``alpha = -0.1`` to contract the boxes by 10%. Only applicable when - a ``model`` and ``patches_field`` are specified + a ``model`` and ``patches_field``/``roi_field`` are specified batch_size (None): an optional batch size to use when computing embeddings. Only applicable when a ``model`` is provided num_workers (None): the number of workers to use when loading images. @@ -660,6 +667,7 @@ def compute_similarity( return fbs.compute_similarity( samples, patches_field, + roi_field, embeddings, brain_key, model, @@ -675,6 +683,111 @@ def compute_similarity( ) +def compute_near_duplicates( + samples, + threshold=0.2, + roi_field=None, + embeddings=None, + similarity_index=None, + model=None, + model_kwargs=None, + force_square=False, + alpha=None, + batch_size=None, + num_workers=None, + skip_failures=True, + progress=None, +): + """Detects potential duplicates in the given sample collection. + + Calling this method only initializes the index. You can then call the + methods exposed on the returned object to perform the following operations: + + - :meth:`duplicate_ids `: + A list of duplicate IDs + + - :meth:`neighbors_map `: + A dictionary mapping IDs to lists of ``(dup_id, dist)`` tuples + + - :meth:`duplicates_view() `: + Returns a view of all duplicates in the input collection + + Args: + samples: a :class:`fiftyone.core.collections.SampleCollection` + threshold (0.2): the similarity distance threshold to use when + detecting duplicates. Values in ``[0.1, 0.25]`` work well for the + default setup + roi_field (None): an optional :class:`fiftyone.core.labels.Detection`, + :class:`fiftyone.core.labels.Detections`, + :class:`fiftyone.core.labels.Polyline`, or + :class:`fiftyone.core.labels.Polylines` field defining a region of + interest within each image to use to compute leaks + embeddings (None): if no ``model`` is provided, this argument specifies + pre-computed embeddings to use, which can be any of the following: + + - a ``num_samples x num_dims`` array of embeddings + - if ``roi_field`` is specified, a dict mapping sample IDs to + ``num_patches x num_dims`` arrays of patch embeddings + - the name of a dataset field containing the embeddings to use + + If a ``model`` is provided, this argument specifies the name of a + field in which to store the computed embeddings. In either case, + when working with patch embeddings, you can provide either the + fully-qualified path to the patch embeddings or just the name of + the label attribute in ``roi_field`` + similarity_index (None): a + :class:`fiftyone.brain.similarity.SimilarityIndex` or the brain key + of a similarity index to use to load pre-computed embeddings + model (None): a :class:`fiftyone.core.models.Model` or the name of a + model from the + `FiftyOne Model Zoo `_ + to use to generate embeddings. The model must expose embeddings + (``model.has_embeddings = True``) + model_kwargs (None): a dictionary of optional keyword arguments to pass + to the model's ``Config`` when a model name is provided + force_square (False): whether to minimally manipulate the patch + bounding boxes into squares prior to extraction. Only applicable + when a ``model`` and ``roi_field`` are specified + alpha (None): an optional expansion/contraction to apply to the patches + before extracting them, in ``[-1, inf)``. If provided, the length + and width of the box are expanded (or contracted, when + ``alpha < 0``) by ``(100 * alpha)%``. For example, set + ``alpha = 0.1`` to expand the boxes by 10%, and set + ``alpha = -0.1`` to contract the boxes by 10%. Only applicable when + a ``model`` and ``roi_field`` are specified + batch_size (None): a batch size to use when computing embeddings. Only + applicable when a ``model`` is provided + num_workers (None): the number of workers to use when loading images. + Only applicable when a Torch-based model is being used to compute + embeddings + skip_failures (True): whether to gracefully continue without raising an + error if embeddings cannot be generated for a sample + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead + + Returns: + a :class:`fiftyone.brain.similarity.SimilarityIndex` + """ + import fiftyone.brain.internal.core.duplicates as fbd + + return fbd.compute_near_duplicates( + samples, + threshold=threshold, + roi_field=roi_field, + embeddings=embeddings, + similarity_index=similarity_index, + model=model, + model_kwargs=model_kwargs, + force_square=force_square, + alpha=alpha, + batch_size=batch_size, + num_workers=num_workers, + skip_failures=skip_failures, + progress=progress, + ) + + def compute_exact_duplicates( samples, num_workers=None, @@ -684,7 +797,7 @@ def compute_exact_duplicates( """Detects duplicate media in a sample collection. This method detects exact duplicates with the same filehash. Use - :meth:`compute_similarity` to detect near-duplicate images. + :meth:`compute_near_duplicates` to detect near-duplicates. If duplicates are found, the first instance in ``samples`` will be the key in the returned dictionary, while the subsequent duplicates will be the @@ -714,10 +827,13 @@ def compute_leaky_splits( samples, splits, threshold=0.2, + roi_field=None, embeddings=None, similarity_index=None, model=None, model_kwargs=None, + force_square=False, + alpha=None, batch_size=None, num_workers=None, skip_failures=True, @@ -752,14 +868,24 @@ def compute_leaky_splits( threshold (0.2): the similarity distance threshold to use when detecting leaks. Values in ``[0.1, 0.25]`` work well for the default setup + roi_field (None): an optional :class:`fiftyone.core.labels.Detection`, + :class:`fiftyone.core.labels.Detections`, + :class:`fiftyone.core.labels.Polyline`, or + :class:`fiftyone.core.labels.Polylines` field defining a region of + interest within each image to use to compute leaks embeddings (None): if no ``model`` is provided, this argument specifies pre-computed embeddings to use, which can be any of the following: - a ``num_samples x num_dims`` array of embeddings + - if ``roi_field`` is specified, a dict mapping sample IDs to + ``num_patches x num_dims`` arrays of patch embeddings - the name of a dataset field containing the embeddings to use If a ``model`` is provided, this argument specifies the name of a - field in which to store the computed embeddings + field in which to store the computed embeddings. In either case, + when working with patch embeddings, you can provide either the + fully-qualified path to the patch embeddings or just the name of + the label attribute in ``roi_field`` similarity_index (None): a :class:`fiftyone.brain.similarity.SimilarityIndex` or the brain key of a similarity index to use to load pre-computed embeddings @@ -770,6 +896,16 @@ def compute_leaky_splits( (``model.has_embeddings = True``) model_kwargs (None): a dictionary of optional keyword arguments to pass to the model's ``Config`` when a model name is provided + force_square (False): whether to minimally manipulate the patch + bounding boxes into squares prior to extraction. Only applicable + when a ``model`` and ``roi_field`` are specified + alpha (None): an optional expansion/contraction to apply to the patches + before extracting them, in ``[-1, inf)``. If provided, the length + and width of the box are expanded (or contracted, when + ``alpha < 0``) by ``(100 * alpha)%``. For example, set + ``alpha = 0.1`` to expand the boxes by 10%, and set + ``alpha = -0.1`` to contract the boxes by 10%. Only applicable when + a ``model`` and ``roi_field`` are specified batch_size (None): a batch size to use when computing embeddings. Only applicable when a ``model`` is provided num_workers (None): the number of workers to use when loading images. @@ -790,10 +926,13 @@ def compute_leaky_splits( samples, splits, threshold=threshold, + roi_field=roi_field, embeddings=embeddings, similarity_index=similarity_index, model=model, model_kwargs=model_kwargs, + force_square=force_square, + alpha=alpha, batch_size=batch_size, num_workers=num_workers, skip_failures=skip_failures, diff --git a/fiftyone/brain/internal/core/duplicates.py b/fiftyone/brain/internal/core/duplicates.py index 44c15d87..aa91a960 100644 --- a/fiftyone/brain/internal/core/duplicates.py +++ b/fiftyone/brain/internal/core/duplicates.py @@ -10,13 +10,87 @@ import logging import multiprocessing +import eta.core.utils as etau + import fiftyone.core.media as fom import fiftyone.core.utils as fou import fiftyone.core.validation as fov +import fiftyone.brain as fb +import fiftyone.brain.similarity as fbs +import fiftyone.brain.internal.core.utils as fbu + logger = logging.getLogger(__name__) +_DEFAULT_MODEL = "resnet18-imagenet-torch" + + +def compute_near_duplicates( + samples, + threshold=None, + roi_field=None, + embeddings=None, + similarity_index=None, + model=None, + model_kwargs=None, + force_square=False, + alpha=None, + batch_size=None, + num_workers=None, + skip_failures=True, + progress=None, +): + """See ``fiftyone/brain/__init__.py``.""" + + fov.validate_collection(samples) + + if etau.is_str(embeddings): + embeddings_field, embeddings_exist = fbu.parse_embeddings_field( + samples, + embeddings, + ) + embeddings = None + else: + embeddings_field = None + embeddings_exist = None + + if etau.is_str(similarity_index): + similarity_index = samples.load_brain_results(similarity_index) + + if ( + model is None + and embeddings is None + and similarity_index is None + and not embeddings_exist + ): + model = _DEFAULT_MODEL + + if similarity_index is None: + similarity_index = fb.compute_similarity( + samples, + backend="sklearn", + roi_field=roi_field, + embeddings=embeddings_field or embeddings, + model=model, + model_kwargs=model_kwargs, + force_square=force_square, + alpha=alpha, + batch_size=batch_size, + num_workers=num_workers, + skip_failures=skip_failures, + progress=progress, + ) + elif not isinstance(similarity_index, fbs.DuplicatesMixin): + raise ValueError( + "This method only supports similarity indexes that implement the " + "%s mixin" % fbs.DuplicatesMixin + ) + + similarity_index.find_duplicates(thresh=threshold) + + return similarity_index + def compute_exact_duplicates(samples, num_workers, skip_failures, progress): """See ``fiftyone/brain/__init__.py``.""" diff --git a/fiftyone/brain/internal/core/elasticsearch.py b/fiftyone/brain/internal/core/elasticsearch.py index 3b45621f..38ba9ff1 100644 --- a/fiftyone/brain/internal/core/elasticsearch.py +++ b/fiftyone/brain/internal/core/elasticsearch.py @@ -37,12 +37,6 @@ class ElasticsearchSimilarityConfig(SimilarityConfig): """Configuration for a Elasticsearch similarity instance. Args: - embeddings_field (None): the sample field containing the embeddings - model (None): the :class:`fiftyone.core.models.Model` or name of the - zoo model that was used to compute embeddings, if known - patches_field (None): the sample field defining the patches being - analyzed, if any - supports_prompts (None): whether this run supports prompt queries index_name (None): the name of the Elasticsearch index to use or create. If none is provided, a new index will be created metric ("cosine"): the embedding distance metric to use when creating a @@ -63,10 +57,6 @@ class ElasticsearchSimilarityConfig(SimilarityConfig): def __init__( self, - embeddings_field=None, - model=None, - patches_field=None, - supports_prompts=None, index_name=None, metric="cosine", hosts=None, @@ -86,13 +76,7 @@ def __init__( % (metric, tuple(_SUPPORTED_METRICS.keys())) ) - super().__init__( - embeddings_field=embeddings_field, - model=model, - patches_field=patches_field, - supports_prompts=supports_prompts, - **kwargs, - ) + super().__init__(**kwargs) self.index_name = index_name self.metric = metric diff --git a/fiftyone/brain/internal/core/lancedb.py b/fiftyone/brain/internal/core/lancedb.py index 3a492898..f15e75aa 100644 --- a/fiftyone/brain/internal/core/lancedb.py +++ b/fiftyone/brain/internal/core/lancedb.py @@ -36,13 +36,6 @@ class LanceDBSimilarityConfig(SimilarityConfig): """Configuration for a LanceDB similarity instance. Args: - embeddings_field (None): the sample field containing the embeddings, - if one was provided - model (None): the :class:`fiftyone.core.models.Model` or name of the - zoo model that was used to compute embeddings, if known - patches_field (None): the sample field defining the patches being - analyzed, if any - supports_prompts (None): whether this run supports prompt queries table_name (None): the name of the LanceDB table to use. If none is provided, a new table will be created metric ("cosine"): the embedding distance metric to use when creating a @@ -53,10 +46,6 @@ class LanceDBSimilarityConfig(SimilarityConfig): def __init__( self, - embeddings_field=None, - model=None, - patches_field=None, - supports_prompts=None, table_name=None, metric="cosine", uri="/tmp/lancedb", @@ -68,13 +57,7 @@ def __init__( % (metric, tuple(_SUPPORTED_METRICS.keys())) ) - super().__init__( - embeddings_field=embeddings_field, - model=model, - patches_field=patches_field, - supports_prompts=supports_prompts, - **kwargs, - ) + super().__init__(**kwargs) self.table_name = table_name self.metric = metric diff --git a/fiftyone/brain/internal/core/leaky_splits.py b/fiftyone/brain/internal/core/leaky_splits.py index a900dab9..7755c06c 100644 --- a/fiftyone/brain/internal/core/leaky_splits.py +++ b/fiftyone/brain/internal/core/leaky_splits.py @@ -30,10 +30,13 @@ def compute_leaky_splits( samples, splits, threshold=None, + roi_field=None, embeddings=None, similarity_index=None, model=None, model_kwargs=None, + force_square=False, + alpha=None, batch_size=None, num_workers=None, skip_failures=True, @@ -41,7 +44,7 @@ def compute_leaky_splits( ): """See ``fiftyone/brain/__init__.py``.""" - fov.validate_image_collection(samples) + fov.validate_collection(samples) if etau.is_str(embeddings): embeddings_field, embeddings_exist = fbu.parse_embeddings_field( @@ -80,9 +83,12 @@ def compute_leaky_splits( similarity_index = fb.compute_similarity( samples, backend="sklearn", + roi_field=roi_field, + embeddings=embeddings_field or embeddings, model=model, model_kwargs=model_kwargs, - embeddings=embeddings_field or embeddings, + force_square=force_square, + alpha=alpha, batch_size=batch_size, num_workers=num_workers, skip_failures=skip_failures, diff --git a/fiftyone/brain/internal/core/milvus.py b/fiftyone/brain/internal/core/milvus.py index 5067630f..6373cf9c 100644 --- a/fiftyone/brain/internal/core/milvus.py +++ b/fiftyone/brain/internal/core/milvus.py @@ -36,13 +36,6 @@ class MilvusSimilarityConfig(SimilarityConfig): """Configuration for the Milvus similarity backend. Args: - embeddings_field (None): the sample field containing the embeddings, - if one was provided - model (None): the :class:`fiftyone.core.models.Model` or name of the - zoo model that was used to compute embeddings, if known - patches_field (None): the sample field defining the patches being - analyzed, if any - supports_prompts (None): whether this run supports prompt queries collection_name (None): the name of a Milvus collection to use or create. If none is provided, a new collection will be created metric ("dotproduct"): the embedding distance metric to use when @@ -64,14 +57,11 @@ class MilvusSimilarityConfig(SimilarityConfig): ca_pem_path (None): a ca.pem path for TLS two-way server_pem_path (None): a server.pem path for TLS one-way server_name (None): the server name, for TLS + **kwargs: keyword arguments for :class:`SimilarityConfig` """ def __init__( self, - embeddings_field=None, - model=None, - patches_field=None, - supports_prompts=None, collection_name=None, metric="dotproduct", consistency_level="Session", @@ -94,13 +84,7 @@ def __init__( % (metric, tuple(_SUPPORTED_METRICS.keys())) ) - super().__init__( - embeddings_field=embeddings_field, - model=model, - patches_field=patches_field, - supports_prompts=supports_prompts, - **kwargs, - ) + super().__init__(**kwargs) self.collection_name = collection_name self.metric = metric diff --git a/fiftyone/brain/internal/core/mongodb.py b/fiftyone/brain/internal/core/mongodb.py index 88e053b6..1a29451b 100644 --- a/fiftyone/brain/internal/core/mongodb.py +++ b/fiftyone/brain/internal/core/mongodb.py @@ -38,12 +38,6 @@ class MongoDBSimilarityConfig(SimilarityConfig): """Configuration for a MongoDB similarity instance. Args: - embeddings_field (None): the sample field containing the embeddings - model (None): the :class:`fiftyone.core.models.Model` or name of the - zoo model that was used to compute embeddings, if known - patches_field (None): the sample field defining the patches being - analyzed, if any - supports_prompts (None): whether this run supports prompt queries index_name (None): the name of the MongoDB vector index to use or create. If none is provided, a new index will be created metric ("cosine"): the embedding distance metric to use when creating a @@ -52,17 +46,8 @@ class MongoDBSimilarityConfig(SimilarityConfig): **kwargs: keyword arguments for :class:`SimilarityConfig` """ - def __init__( - self, - embeddings_field=None, - model=None, - patches_field=None, - supports_prompts=None, - index_name=None, - metric="cosine", - **kwargs, - ): - if embeddings_field is None and index_name is None: + def __init__(self, index_name=None, metric="cosine", **kwargs): + if kwargs.get("embeddings_field") is None and index_name is None: raise ValueError( "You must provide either the name of a field to read/write " "embeddings for this index by passing the `embeddings` " @@ -72,7 +57,7 @@ def __init__( # @todo support this. Will likely require copying embeddings to a new # collection as vector search indexes do not yet support array fields - if patches_field is not None: + if kwargs.get("patches_field") is not None: raise ValueError( "The MongoDB backend does not yet support patch embeddings" ) @@ -83,13 +68,7 @@ def __init__( % (metric, tuple(_SUPPORTED_METRICS.keys())) ) - super().__init__( - embeddings_field=embeddings_field, - model=model, - patches_field=patches_field, - supports_prompts=supports_prompts, - **kwargs, - ) + super().__init__(**kwargs) self.index_name = index_name self.metric = metric diff --git a/fiftyone/brain/internal/core/pinecone.py b/fiftyone/brain/internal/core/pinecone.py index f9cd74b2..e48ac6af 100644 --- a/fiftyone/brain/internal/core/pinecone.py +++ b/fiftyone/brain/internal/core/pinecone.py @@ -31,13 +31,6 @@ class PineconeSimilarityConfig(SimilarityConfig): """Configuration for the Pinecone similarity backend. Args: - embeddings_field (None): the sample field containing the embeddings, - if one was provided - model (None): the :class:`fiftyone.core.models.Model` or name of the - zoo model that was used to compute embeddings, if known - patches_field (None): the sample field defining the patches being - analyzed, if any - supports_prompts (None): whether this run supports prompt queries index_name (None): the name of a Pinecone index to use or create. If none is provided, a new index will be created index_type (None): the index type to use when creating a new index. @@ -61,14 +54,11 @@ class PineconeSimilarityConfig(SimilarityConfig): region (None): a region to use when creating serverless indexes environment (None): an environment to use when creating pod-based indexes + **kwargs: keyword arguments for :class:`SimilarityConfig` """ def __init__( self, - embeddings_field=None, - model=None, - patches_field=None, - supports_prompts=None, index_name=None, index_type=None, namespace=None, @@ -89,13 +79,7 @@ def __init__( % (metric, _SUPPORTED_METRICS) ) - super().__init__( - embeddings_field=embeddings_field, - model=model, - patches_field=patches_field, - supports_prompts=supports_prompts, - **kwargs, - ) + super().__init__(**kwargs) self.index_name = index_name self.index_type = index_type diff --git a/fiftyone/brain/internal/core/qdrant.py b/fiftyone/brain/internal/core/qdrant.py index 966e6f0d..6c063d0c 100644 --- a/fiftyone/brain/internal/core/qdrant.py +++ b/fiftyone/brain/internal/core/qdrant.py @@ -36,13 +36,6 @@ class QdrantSimilarityConfig(SimilarityConfig): """Configuration for the Qdrant similarity backend. Args: - embeddings_field (None): the sample field containing the embeddings, - if one was provided - model (None): the :class:`fiftyone.core.models.Model` or name of the - zoo model that was used to compute embeddings, if known - patches_field (None): the sample field defining the patches being - analyzed, if any - supports_prompts (None): whether this run supports prompt queries collection_name (None): the name of a Qdrant collection to use or create. If none is provided, a new collection will be created metric (None): the embedding distance metric to use when creating a @@ -64,14 +57,11 @@ class QdrantSimilarityConfig(SimilarityConfig): api_key (None): a Qdrant API key to use grpc_port (None): Port of Qdrant gRPC interface prefer_grpc (None): If `true`, use gRPC interface when possible + **kwargs: keyword arguments for :class:`SimilarityConfig` """ def __init__( self, - embeddings_field=None, - model=None, - patches_field=None, - supports_prompts=None, collection_name=None, metric=None, replication_factor=None, @@ -92,13 +82,7 @@ def __init__( % (metric, tuple(_SUPPORTED_METRICS.keys())) ) - super().__init__( - embeddings_field=embeddings_field, - model=model, - patches_field=patches_field, - supports_prompts=supports_prompts, - **kwargs, - ) + super().__init__(**kwargs) self.collection_name = collection_name self.metric = metric diff --git a/fiftyone/brain/internal/core/redis.py b/fiftyone/brain/internal/core/redis.py index 0771c281..53d35ae4 100644 --- a/fiftyone/brain/internal/core/redis.py +++ b/fiftyone/brain/internal/core/redis.py @@ -35,13 +35,6 @@ class RedisSimilarityConfig(SimilarityConfig): """Configuration for the Redis similarity backend. Args: - embeddings_field (None): the sample field containing the embeddings, - if one was provided - model (None): the :class:`fiftyone.core.models.Model` or name of the - zoo model that was used to compute embeddings, if known - patches_field (None): the sample field defining the patches being - analyzed, if any - supports_prompts (None): whether this run supports prompt queries index_name (None): the name of a Redis index to use or create. If none is provided, a new index will be created metric ("cosine"): the embedding distance metric to use when creating a @@ -54,14 +47,11 @@ class RedisSimilarityConfig(SimilarityConfig): db (0): the database to use username (None): a username to use password (None): a password to use + **kwargs: keyword arguments for :class:`SimilarityConfig` """ def __init__( self, - embeddings_field=None, - model=None, - patches_field=None, - supports_prompts=None, index_name=None, metric="cosine", algorithm="FLAT", @@ -78,13 +68,7 @@ def __init__( % (metric, tuple(_SUPPORTED_METRICS.keys())) ) - super().__init__( - embeddings_field=embeddings_field, - model=model, - patches_field=patches_field, - supports_prompts=supports_prompts, - **kwargs, - ) + super().__init__(**kwargs) self.index_name = index_name self.metric = metric diff --git a/fiftyone/brain/internal/core/representativeness.py b/fiftyone/brain/internal/core/representativeness.py index 2ddfd04f..800da428 100644 --- a/fiftyone/brain/internal/core/representativeness.py +++ b/fiftyone/brain/internal/core/representativeness.py @@ -63,7 +63,7 @@ def compute_representativeness( # ranking and points on the outliers with low ranking. # - fov.validate_image_collection(samples) + fov.validate_collection(samples) if roi_field is not None: fov.validate_collection_label_fields( diff --git a/fiftyone/brain/internal/core/sklearn.py b/fiftyone/brain/internal/core/sklearn.py index 47d65227..7508d2a7 100644 --- a/fiftyone/brain/internal/core/sklearn.py +++ b/fiftyone/brain/internal/core/sklearn.py @@ -40,33 +40,13 @@ class SklearnSimilarityConfig(SimilarityConfig): """Configuration for the sklearn similarity backend. Args: - embeddings_field (None): the sample field containing the embeddings, - if one was provided - model (None): the :class:`fiftyone.core.models.Model` or name of the - zoo model that was used to compute embeddings, if known - patches_field (None): the sample field defining the patches being - analyzed, if any - supports_prompts (None): whether this run supports prompt queries metric ("cosine"): the embedding distance metric to use. See ``sklearn.metrics.pairwise_distance`` for supported values + **kwargs: keyword arguments for :class:`SimilarityConfig` """ - def __init__( - self, - embeddings_field=None, - model=None, - patches_field=None, - supports_prompts=None, - metric="cosine", - **kwargs, - ): - super().__init__( - embeddings_field=embeddings_field, - model=model, - patches_field=patches_field, - supports_prompts=supports_prompts, - **kwargs, - ) + def __init__(self, metric="cosine", **kwargs): + super().__init__(**kwargs) self.metric = metric @property diff --git a/fiftyone/brain/internal/core/uniqueness.py b/fiftyone/brain/internal/core/uniqueness.py index e2682ddc..7b89077d 100644 --- a/fiftyone/brain/internal/core/uniqueness.py +++ b/fiftyone/brain/internal/core/uniqueness.py @@ -64,7 +64,7 @@ def compute_uniqueness( # to dense clusters of related samples. # - fov.validate_image_collection(samples) + fov.validate_collection(samples) if roi_field is not None: fov.validate_collection_label_fields( diff --git a/fiftyone/brain/similarity.py b/fiftyone/brain/similarity.py index f367f972..e888df0a 100644 --- a/fiftyone/brain/similarity.py +++ b/fiftyone/brain/similarity.py @@ -32,6 +32,13 @@ logger = logging.getLogger(__name__) +_ALLOWED_ROI_FIELD_TYPES = ( + fol.Detection, + fol.Detections, + fol.Polyline, + fol.Polylines, +) + _DEFAULT_MODEL = "mobilenet-v2-imagenet-torch" _DEFAULT_BATCH_SIZE = None @@ -39,6 +46,7 @@ def compute_similarity( samples, patches_field, + roi_field, embeddings, brain_key, model, @@ -56,6 +64,11 @@ def compute_similarity( fov.validate_collection(samples) + if roi_field is not None: + fov.validate_collection_label_fields( + samples, roi_field, _ALLOWED_ROI_FIELD_TYPES + ) + # Allow for `embeddings_field=XXX` and `embeddings=False` together embeddings_field = kwargs.pop("embeddings_field", None) if embeddings_field is not None or etau.is_str(embeddings): @@ -66,7 +79,7 @@ def compute_similarity( embeddings_field, embeddings_exist = fbu.parse_embeddings_field( samples, embeddings_field, - patches_field=patches_field, + patches_field=patches_field or roi_field, ) else: embeddings_field = None @@ -92,6 +105,7 @@ def compute_similarity( backend, embeddings_field=embeddings_field, patches_field=patches_field, + roi_field=roi_field, model=model, model_kwargs=model_kwargs, supports_prompts=supports_prompts, @@ -118,14 +132,23 @@ def compute_similarity( if not embeddings_exist: embeddings_field = None + if roi_field is not None: + handle_missing = "image" + agg_fcn = lambda e: np.mean(e, axis=0) + else: + handle_missing = "skip" + agg_fcn = None + embeddings, sample_ids, label_ids = fbu.get_embeddings( samples, model=_model, - patches_field=patches_field, + patches_field=patches_field or roi_field, embeddings=embeddings, embeddings_field=embeddings_field, force_square=force_square, alpha=alpha, + handle_missing=handle_missing, + agg_fcn=agg_fcn, batch_size=batch_size, num_workers=num_workers, skip_failures=skip_failures, @@ -186,6 +209,8 @@ class SimilarityConfig(fob.BrainMethodConfig): to the model's ``Config`` when a model name is provided patches_field (None): the sample field defining the patches being analyzed, if any + roi_field (None): the sample field defining a region of interest within + each image to use to compute embeddings, if any supports_prompts (False): whether this run supports prompt queries """ @@ -195,6 +220,7 @@ def __init__( model=None, model_kwargs=None, patches_field=None, + roi_field=None, supports_prompts=None, **kwargs, ): @@ -205,6 +231,7 @@ def __init__( self.model = model self.model_kwargs = model_kwargs self.patches_field = patches_field + self.roi_field = roi_field self.supports_prompts = supports_prompts super().__init__(**kwargs) @@ -938,12 +965,23 @@ def compute_embeddings( "This index does not support skipping existing IDs" ) + if self.config.roi_field is not None: + patches_field = self.config.roi_field + handle_missing = "image" + agg_fcn = lambda e: np.mean(e, axis=0) + else: + patches_field = self.config.patches_field + handle_missing = "skip" + agg_fcn = None + return fbu.get_embeddings( samples, model=model, - patches_field=self.config.patches_field, + patches_field=patches_field, force_square=force_square, alpha=alpha, + handle_missing=handle_missing, + agg_fcn=agg_fcn, batch_size=batch_size, num_workers=num_workers, skip_failures=skip_failures,