test on multiple

Author: IlyasMoutawwakil

.github/workflows/test_pipelines.yml

@@ -11,15 +11,17 @@ concurrency:
   cancel-in-progress: true
 
 env:
+  UV_SYSTEM_PYTHON: 1
+  UV_TORCH_BACKEND: auto
   TRANSFORMERS_IS_CI: true
 
 jobs:
   build:
     strategy:
       fail-fast: false
       matrix:
-        runs-on: [ubuntu-22.04]
         python-version: [3.9]
+        runs-on: [ubuntu-22.04, windows-2022, macos-14]
 
     runs-on: ${{ matrix.runs-on }}
 
@@ -34,10 +36,9 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install --upgrade pip
-          pip install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-          pip install --no-cache-dir optimum-onnx[onnxruntime]@git+https://github.com/huggingface/optimum-onnx.git
-          pip install .[tests]
+          pip install --upgrade pip uv
+          uv pip install --no-cache-dir optimum-onnx[onnxruntime]@git+https://github.com/huggingface/optimum-onnx.git
+          uv pip install --no-cache-dir .[tests]
 
       - name: Test with pytest
         run: |

tests/pipelines/test_pipelines.py

@@ -17,11 +17,16 @@
 from typing import Any, Dict
 
 import numpy as np
+from huggingface_hub.constants import HF_HUB_CACHE
 from PIL import Image
 from transformers import AutoTokenizer
 from transformers.pipelines import Pipeline
 
 from optimum.pipelines import pipeline
+from optimum.utils.testing_utils import remove_directory
+
+
+GENERATE_KWARGS = {"max_new_tokens": 10, "min_new_tokens": 5, "do_sample": True}
 
 
 class ORTPipelineTest(unittest.TestCase):
@@ -33,13 +38,11 @@ def _create_dummy_text(self) -> str:
 
     def _create_dummy_image(self) -> Image.Image:
         """Create dummy image input for image-based tasks"""
-        # Create a simple RGB image
         np_image = np.random.randint(0, 256, (224, 224, 3), dtype=np.uint8)
         return Image.fromarray(np_image)
 
     def _create_dummy_audio(self) -> Dict[str, Any]:
         """Create dummy audio input for audio-based tasks"""
-        # Create a dummy audio array (16kHz sample rate, 1 second)
         sample_rate = 16000
         audio_array = np.random.randn(sample_rate).astype(np.float32)
         return {"array": audio_array, "sampling_rate": sample_rate}
@@ -111,7 +114,7 @@ def test_text_generation_pipeline(self):
         pipe = pipeline(task="text-generation", accelerator="ort")
         self.assertIsInstance(pipe, Pipeline)
         text = "The future of AI is"
-        result = pipe(text, max_new_tokens=50, do_sample=False)
+        result = pipe(text, **GENERATE_KWARGS)
 
         self.assertIsInstance(result, list)
         self.assertGreater(len(result), 0)
@@ -123,7 +126,7 @@ def test_summarization_pipeline(self):
         pipe = pipeline(task="summarization", accelerator="ort")
         self.assertIsInstance(pipe, Pipeline)
         text = "The quick brown fox jumps over the lazy dog."
-        result = pipe(text, max_new_tokens=50, min_new_tokens=10, do_sample=False)
+        result = pipe(text, **GENERATE_KWARGS)
 
         self.assertIsInstance(result, list)
         self.assertGreater(len(result), 0)
@@ -134,7 +137,7 @@ def test_translation_pipeline(self):
         pipe = pipeline(task="translation_en_to_de", accelerator="ort")
         self.assertIsInstance(pipe, Pipeline)
         text = "Hello, how are you?"
-        result = pipe(text, max_new_tokens=50)
+        result = pipe(text, **GENERATE_KWARGS)
 
         self.assertIsInstance(result, list)
         self.assertGreater(len(result), 0)
@@ -145,7 +148,7 @@ def test_text2text_generation_pipeline(self):
         pipe = pipeline(task="text2text-generation", accelerator="ort")
         self.assertIsInstance(pipe, Pipeline)
         text = "translate English to German: Hello, how are you?"
-        result = pipe(text, max_new_tokens=50)
+        result = pipe(text, **GENERATE_KWARGS)
 
         self.assertIsInstance(result, list)
         self.assertGreater(len(result), 0)
@@ -194,7 +197,7 @@ def test_image_to_text_pipeline(self):
         pipe = pipeline(task="image-to-text", accelerator="ort")
         self.assertIsInstance(pipe, Pipeline)
         image = self._create_dummy_image()
-        result = pipe(image)
+        result = pipe(image, generate_kwargs=GENERATE_KWARGS)
 
         self.assertIsInstance(result, list)
         self.assertGreater(len(result), 0)
@@ -214,7 +217,7 @@ def test_image_to_image_pipeline(self):
     #     """Test automatic speech recognition ORT pipeline"""
     #     pipe = pipeline(task="automatic-speech-recognition", accelerator="ort")
     #     audio = self._create_dummy_audio()
-    #     result = pipe(audio)
+    #     result = pipe(audio, generate_kwargs=GENERATE_KWARGS)
 
     #     self.assertIsInstance(result, dict)
     #     self.assertIn("text", result)
@@ -246,7 +249,7 @@ def test_pipeline_with_ort_model(self):
         self.assertIsInstance(result[0], list)
         self.assertIsInstance(result[0][0], list)
 
-    def test_pipeline_with_custom_model_id(self):
+    def test_pipeline_with_model_id(self):
         """Test ORT pipeline with a custom model id"""
         pipe = pipeline(task="feature-extraction", model="distilbert-base-cased", accelerator="ort")
         self.assertIsInstance(pipe, Pipeline)
@@ -265,9 +268,12 @@ def test_pipeline_with_invalid_task(self):
     def test_pipeline_with_invalid_accelerator(self):
         """Test ORT pipeline with an unsupported accelerator"""
         with self.assertRaises(ValueError) as context:
-            _ = pipeline(task="text-classification", accelerator="invalid-accelerator")
+            _ = pipeline(task="feature-extraction", accelerator="invalid-accelerator")
         self.assertIn("Accelerator invalid-accelerator not recognized", str(context.exception))
 
+    def tearDown(self):
+        remove_directory(HF_HUB_CACHE)
+
 
 if __name__ == "__main__":
     unittest.main()