diff --git a/src/scikit_build_core/builder/builder.py b/src/scikit_build_core/builder/builder.py
index db11bb4c..e35f8028 100644
--- a/src/scikit_build_core/builder/builder.py
+++ b/src/scikit_build_core/builder/builder.py
@@ -138,6 +138,10 @@ def configure(
             self.config.prefix_dirs.append(DIR.parent.parent)
             logger.debug("Extra SITE_PACKAGES: {}", site_packages)
 
+        # Add site-packages to ignore prefix root
+        # TODO: Should we try to add the installation wheel.install-dir + wheel.packages here?
+        self.config.ignore_prefix_dirs.append(site_packages)
+
         # Add the FindPython backport if needed
         if self.config.cmake.version < self.settings.backport.find_python:
             fp_dir = Path(find_python.__file__).parent.resolve()
@@ -158,6 +162,7 @@ def configure(
         if name is not None:
             canonical_name = name.replace("-", "_").replace(".", "_")
             cache_config["SKBUILD_PROJECT_NAME"] = canonical_name
+            self.config.canonical_name = canonical_name
         if version is not None:
             cache_config["SKBUILD_PROJECT_VERSION"] = ".".join(
                 str(v) for v in version.release
diff --git a/src/scikit_build_core/cmake.py b/src/scikit_build_core/cmake.py
index b9b0ac36..ab4a12d2 100644
--- a/src/scikit_build_core/cmake.py
+++ b/src/scikit_build_core/cmake.py
@@ -12,6 +12,8 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
+from packaging.version import Version
+
 from . import __version__
 from ._logging import logger
 from ._shutil import Run
@@ -22,7 +24,6 @@
     from collections.abc import Generator, Iterable, Mapping, Sequence
 
     from packaging.specifiers import SpecifierSet
-    from packaging.version import Version
 
     from ._compat.typing import Self
 
@@ -79,6 +80,8 @@ class CMaker:
     build_type: str
     module_dirs: list[Path] = dataclasses.field(default_factory=list)
     prefix_dirs: list[Path] = dataclasses.field(default_factory=list)
+    ignore_prefix_dirs: list[Path] = dataclasses.field(default_factory=list)
+    canonical_name: str = ""
     init_cache_file: Path = dataclasses.field(init=False, default=Path())
     env: dict[str, str] = dataclasses.field(init=False, default_factory=os.environ.copy)
     single_config: bool = not sysconfig.get_platform().startswith("win")
@@ -162,6 +165,23 @@ def init_cache(
                 )
                 f.write('set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE "BOTH" CACHE PATH "")\n')
 
+            if self.ignore_prefix_dirs:
+                ignore_prefix_dirs_str = ""
+                for ignore_prefix in self.ignore_prefix_dirs:
+                    # TODO: Detect the appropriate prefix dirs to ignore depending on installed packages?
+                    ignore_prefix_dirs_str += f"{ignore_prefix / self.canonical_name};"
+                ignore_prefix_dirs_str = ignore_prefix_dirs_str.strip(";").replace(
+                    "\\", "/"
+                )
+                ignore_path_var = (
+                    "CMAKE_IGNORE_PREFIX_PATH"
+                    if self.cmake.version >= Version("3.23")
+                    else "CMAKE_IGNORE_PATH"
+                )
+                f.write(
+                    f'set({ignore_path_var} [===[{ignore_prefix_dirs_str}]===] CACHE PATH "" FORCE)\n'
+                )
+
         contents = self.init_cache_file.read_text(encoding="utf-8").strip()
         logger.debug(
             "{}:\n{}",