[BE] restructure tests and assets folders

ghstack-source-id: 847008cf86e4cbeb77a00bf1cd455543899fe2b1
Pull Request resolved: #748

Author: tianyu-l

.github/workflows/integration_test_4gpu.yaml

@@ -43,4 +43,4 @@ jobs:
         python -m pip install -e .
 
         mkdir artifacts-to-be-uploaded
-        python ./test_runner.py artifacts-to-be-uploaded --ngpu 4
+        python ./tests/integration_tests.py artifacts-to-be-uploaded --ngpu 4

.github/workflows/integration_test_8gpu.yaml

@@ -38,4 +38,4 @@ jobs:
 
         python -m pip install --force-reinstall --pre torch --index-url https://download.pytorch.org/whl/nightly/cu124
         mkdir artifacts-to-be-uploaded
-        python ./test_runner.py artifacts-to-be-uploaded --ngpu 8
+        python ./tests/integration_tests.py artifacts-to-be-uploaded --ngpu 8

.github/workflows/unit_test_cpu.yaml

@@ -25,4 +25,4 @@ jobs:
         pip config --user set global.progress_bar off
 
         pip install --force-reinstall --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
-        pytest test --cov=. --cov-report=xml --durations=20 -vv
+        pytest tests/unit_tests --cov=. --cov-report=xml --durations=20 -vv

.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
         files: \.py$
         args:
         - --license-filepath
-        - docs/license_header.txt
+        - assets/license_header.txt
 
 -   repo: https://github.com/pycqa/flake8
     rev: 34cbf8ef3950f43d09b85e2e45c15ae5717dc37b

assets/images/readme.md

@@ -36,7 +36,7 @@ dev = [
 ]
 
 [tool.setuptools.dynamic]
-version = {file = "version.txt"}
+version = {file = "assets/version.txt"}
 
 
 # ---- Explicit project build information ---- #

docs/license_header.txt renamed to assets/license_header.txt

@@ -254,7 +254,7 @@ def build_test_list():
                 [
                     "--experimental.pipeline_parallel_degree 2",
                     "--experimental.pipeline_parallel_schedule PipelineScheduleMulti",
-                    "--experimental.pipeline_parallel_schedule_csv ./test/assets/custom_schedule.csv",
+                    "--experimental.pipeline_parallel_schedule_csv ./tests/assets/custom_schedule.csv",
                     "--experimental.pipeline_parallel_microbatches 8",
                 ],
             ],

version.txt renamed to assets/version.txt

@@ -12,7 +12,10 @@
     VisionEncoder,
 )
 
-from test.multimodal_model.test_utils import fixed_init_model, fixed_init_tensor
+from tests.unit_tests.multimodal_model.test_utils import (
+    fixed_init_model,
+    fixed_init_tensor,
+)
 
 
 @pytest.fixture

pyproject.toml

@@ -9,10 +9,10 @@
 from torchtitan.datasets.tokenizer import build_tokenizer
 
 
-class TestCheckpoint:
+class TestDatasetCheckpointing:
     def test_c4_resumption(self):
         dataset_name = "c4_test"
-        dataset_path = "./test/assets/c4_test"
+        dataset_path = "./tests/assets/c4_test"
         batch_size = 1
         seq_len = 1024
         world_size = 4
@@ -41,8 +41,7 @@ def test_c4_resumption(self):
     def _build_dataloader(
         self, dataset_name, dataset_path, batch_size, seq_len, world_size, rank
     ):
-        tokenizer_type = "tiktoken"
-        tokenizer = build_tokenizer("tiktoken", "./test/assets/test_tiktoken.model")
+        tokenizer = build_tokenizer("tiktoken", "./tests/assets/test_tiktoken.model")
         return build_hf_data_loader(
             dataset_name=dataset_name,
             dataset_path=dataset_path,

test/__init__.py renamed to tests/__init__.py

@@ -45,7 +45,7 @@ class DatasetConfig:
         text_processor=_process_c4_text,
     ),
     "c4_test": DatasetConfig(
-        path="test/assets/c4_test",
+        path="tests/assets/c4_test",
         loader=lambda path: load_dataset(path, split="train"),
         text_processor=_process_c4_text,
     ),

test/assets/c4_test/data.json renamed to tests/assets/c4_test/data.json

@@ -24,7 +24,7 @@ name = "llama3"
 flavor = "debugmodel"
 norm_type = "rmsnorm"  # layernorm / np_layernorm / rmsnorm / fused_rmsnorm
 # test tokenizer.model, for debug purpose only
-tokenizer_path = "./test/assets/test_tiktoken.model"
+tokenizer_path = "./tests/assets/test_tiktoken.model"
 
 [optimizer]
 name = "AdamW"