standard tests list generator (#1264)

* standard tests list generator Signed-off-by: Samhita Alla <[email protected]> * bash Signed-off-by: Samhita Alla <[email protected]> * nit Signed-off-by: Samhita Alla <[email protected]> * nit Signed-off-by: Samhita Alla <[email protected]> * add torch Signed-off-by: Samhita Alla <[email protected]> * add torch Signed-off-by: Samhita Alla <[email protected]> * validate tests Signed-off-by: Samhita Alla <[email protected]> * update validation logic Signed-off-by: Samhita Alla <[email protected]> * lint Signed-off-by: Samhita Alla <[email protected]> * debug Signed-off-by: Samhita Alla <[email protected]> * debug Signed-off-by: Samhita Alla <[email protected]> * debug Signed-off-by: Samhita Alla <[email protected]> * debug Signed-off-by: Samhita Alla <[email protected]> * debug Signed-off-by: Samhita Alla <[email protected]> * debug Signed-off-by: Samhita Alla <[email protected]> * modify regex Signed-off-by: Samhita Alla <[email protected]> * debug Signed-off-by: Samhita Alla <[email protected]> * remove re Signed-off-by: Samhita Alla <[email protected]> * remove test.py Signed-off-by: Samhita Alla <[email protected]> * debug Signed-off-by: Samhita Alla <[email protected]> * debug Signed-off-by: Samhita Alla <[email protected]> * debug Signed-off-by: Samhita Alla <[email protected]> * remove test.py Signed-off-by: Samhita Alla <[email protected]> * debug Signed-off-by: Samhita Alla <[email protected]> * change regex Signed-off-by: Samhita Alla <[email protected]> * debug Signed-off-by: Samhita Alla <[email protected]> * add env Signed-off-by: Samhita Alla <[email protected]> * do not truncate output Signed-off-by: Samhita Alla <[email protected]> * search Signed-off-by: Samhita Alla <[email protected]> * clean string Signed-off-by: Samhita Alla <[email protected]> * cleanup Signed-off-by: Samhita Alla <[email protected]> * incorporate suggestions by Eduardo Signed-off-by: Samhita Alla <[email protected]> * remove comment Signed-off-by: Samhita Alla <[email protected]> --------- Signed-off-by: Samhita Alla <[email protected]>
flyteorg · Dec 6, 2023 · ae775d5 · ae775d5
1 parent aa8c72e
commit ae775d5
Show file tree

Hide file tree

Showing 6 changed files with 222 additions and 90 deletions.
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -122,10 +122,17 @@ jobs:
     name: Publish artifacts to github release
     runs-on: ubuntu-latest
     needs: [prerelease]
+    strategy:
+      matrix:
+        python-version: ["3.11"]
     steps:
       - uses: actions/checkout@v2
         with:
           fetch-depth: "0"
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
       - name: setup download artifact dir
         run: |
           mkdir download-artifact
@@ -140,6 +147,7 @@ jobs:
           for i in */; do tar -czvf "../release-snacks/${i%/}.tar.gz" "$i" & done; wait
           cd .. && sudo rm -rf download-artifact/
           cp flyte_tests_manifest.json release-snacks/flyte_tests_manifest.json
+          cp flyte_tests.txt release-snacks/flyte_tests.txt
       - name: Release test manifest
         uses: goreleaser/goreleaser-action@v2
         with:
@@ -203,35 +211,25 @@ jobs:
       - name: Install Python dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install flytekit flytekitplugins-deck-standard
+          pip install flytekit flytekitplugins-deck-standard torch
           pip freeze
       - name: Checkout flytesnacks
         uses: actions/checkout@v3
         with:
           repository: flyteorg/flytesnacks
           path: flytesnacks
+      - name: Verify existence of the tests
+        run: |
+          python flyte_tests_validate.py
       - name: Register specific tests
         run: |
-          for f in \
-                  basics/basics/hello_world.py \
-                  basics/basics/workflow.py \
-                  basics/basics/named_outputs.py \
-                  advanced_composition/advanced_composition/chain_entities.py \
-                  advanced_composition/advanced_composition/dynamics.py \
-                  advanced_composition/advanced_composition/map_task.py \
-                  advanced_composition/advanced_composition/subworkflows.py \
-                  data_types_and_io/data_types_and_io/dataclass.py \
-                  data_types_and_io/data_types_and_io/structured_dataset.py ;
+          while read -r line;
           do
-              # TODO: unpin version of flytekit once the FlyteFile bug is fixed
               pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                   register \
                   --project flytesnacks \
                   --domain development \
-                  --image cr.flyte.org/flyteorg/flytekit:py3.11-1.10.0 \
+                  --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
                   --version ${{ env.FLYTESNACKS_VERSION }} \
-                  flytesnacks/examples/$f;
-          done
-      - name: End2End
-        run: |
-          make end2end_execute
+                  flytesnacks/$line;
+          done < flyte_tests.txt
diff --git a/examples/advanced_composition/advanced_composition/merge_sort.py b/examples/advanced_composition/advanced_composition/merge_sort.py
@@ -29,6 +29,7 @@
 # %% [markdown]
 # A simple split function that divides a list into two halves.
 
+
 # %%
 @task
 def split(numbers: typing.List[int]) -> Tuple[typing.List[int], typing.List[int], int, int]:
@@ -65,6 +66,7 @@ def merge(sorted_list1: typing.List[int], sorted_list2: typing.List[int]) -> typ
 # Generally speaking, the algorithm will recurse through the list, splitting it in half until it reaches a size that we
 # know is efficient enough to run locally. At which point it'll just use the python-builtin sorted function.
 
+
 # %% [markdown]
 # This runs the sorting completely locally. It's faster and more efficient to do so if the entire list fits in memory.
 # %%

diff --git a/examples/advanced_composition/advanced_composition/subworkflows.py b/examples/advanced_composition/advanced_composition/subworkflows.py
@@ -87,6 +87,7 @@ def parent_wf(a: int) -> Tuple[int, str, str]:
 if __name__ == "__main__":
     print(f"Running parent_wf(a=3) {parent_wf(a=3)}")
 
+
 # %% [markdown]
 # Interestingly, we can nest a workflow that has a subworkflow within a workflow.
 # Workflows can be simply composed from other workflows, even if they are standalone entities. Each of the
@@ -164,6 +165,7 @@ def ext_workflow(my_input: str) -> Dict:
     "parent_workflow_execution",
 )
 
+
 # %% [markdown]
 # Define another task that returns the repeated keys (in our case, words) from a dictionary.
 # %%

diff --git a/flyte_tests.txt b/flyte_tests.txt
@@ -0,0 +1,17 @@
+examples/advanced_composition/advanced_composition/chain_entities.py
+examples/advanced_composition/advanced_composition/conditions.py
+examples/advanced_composition/advanced_composition/decorating_tasks.py
+examples/advanced_composition/advanced_composition/decorating_workflows.py
+examples/advanced_composition/advanced_composition/dynamics.py
+examples/advanced_composition/advanced_composition/map_task.py
+examples/advanced_composition/advanced_composition/waiting_for_external_inputs.py
+examples/basics/basics/documenting_workflows.py
+examples/basics/basics/hello_world.py
+examples/basics/basics/named_outputs.py
+examples/basics/basics/shell_task.py
+examples/basics/basics/workflow.py
+examples/data_types_and_io/data_types_and_io/dataclass.py
+examples/data_types_and_io/data_types_and_io/enum_type.py
+examples/data_types_and_io/data_types_and_io/file.py
+examples/data_types_and_io/data_types_and_io/folder.py
+examples/data_types_and_io/data_types_and_io/structured_dataset.py