diff --git a/fiftyone/brain/similarity.py b/fiftyone/brain/similarity.py
index a1789c13..940272c1 100644
--- a/fiftyone/brain/similarity.py
+++ b/fiftyone/brain/similarity.py
@@ -1181,9 +1181,13 @@ def find_duplicates(self, thresh=None, fraction=None):
                 unique_view = self._samples.select(unique_ids)
 
             with self.use_view(unique_view):
-                nearest_ids, dists = self._kneighbors(
+                _sample_ids, _label_ids, dists = self._kneighbors(
                     query=duplicate_ids, k=1, return_dists=True
                 )
+                if self.config.patches_field is not None:
+                    nearest_ids = _label_ids
+                else:
+                    nearest_ids = _sample_ids
 
             neighbors_map = defaultdict(list)
             for dup_id, _ids, _dists in zip(duplicate_ids, nearest_ids, dists):