diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml
index 68fe1f0d2042..18cf047e2c31 100644
--- a/.github/workflows/push_tests.yml
+++ b/.github/workflows/push_tests.yml
@@ -40,7 +40,6 @@ jobs:
         python -m pip uninstall -y torch torchvision torchtext
         python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cu116
         python -m pip install -e .[quality,test]
-        python -m pip install scipy transformers
 
     - name: Environment
       run: |
diff --git a/setup.py b/setup.py
index 5c39e3bc5a6b..d3498a87c170 100644
--- a/setup.py
+++ b/setup.py
@@ -168,7 +168,7 @@ def run(self):
 extras["quality"] = ["black==22.3", "isort>=5.5.4", "flake8>=3.8.3", "hf-doc-builder"]
 extras["docs"] = ["hf-doc-builder"]
 extras["training"] = ["accelerate", "datasets", "tensorboard", "modelcards"]
-extras["test"] = ["pytest", "pytest-timeout", "pytest-xdist"]
+extras["test"] = ["datasets", "pytest", "pytest-timeout", "pytest-xdist", "scipy", "transformers"]
 extras["dev"] = extras["quality"] + extras["test"] + extras["training"] + extras["docs"]
 
 install_requires = [