diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 31351632e..aa88261d7 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -212,7 +212,7 @@ jobs:
         run: python -c "import sys; print(sys.version)"
       - name: Install tox
         run: |
-          uv tool install tox --with=tox-gh-actions --with=tox-uv --with=pip
+          uv tool install tox --with=tox-gh-actions --with=tox-uv
       - name: Show tox config
         run: tox c
       - name: Run tox
diff --git a/pyproject.toml b/pyproject.toml
index b62c0048a..f307d1d8c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -200,3 +200,6 @@ enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
 [tool.codespell]
 skip = "*/data/*,./nibabel-data"
 ignore-words-list = "ans,te,ue,ist,nin,nd,ccompiler,ser"
+
+[tool.uv.pip]
+only-binary = ["numpy", "scipy", "h5py"]
diff --git a/tox.ini b/tox.ini
index bd83a79db..236186f72 100644
--- a/tox.ini
+++ b/tox.ini
@@ -71,6 +71,7 @@ set_env =
   dev: PIP_ONLY_BINARY=numpy,scipy,h5py
   !dev: PIP_ONLY_BINARY=numpy,scipy,h5py,pillow,matplotlib
   pre,dev: PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
+  pre,dev: UV_INDEX=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
 extras = test
 deps =
   # General minimum dependencies: pin based on API usage