[Tests] Managed spot test option (skypilot-org#1611)

* Revert setup failure

* add option for --managed-spot

* remove exception raising

* format

* format

* address comment

* comments

Author: Michaelvll

CONTRIBUTING.md

@@ -35,6 +35,9 @@ pytest --lf
 # Run one of the smoke tests
 pytest tests/test_smoke.py::test_minimal
 
+# Only run managed spot tests
+pytest tests/test_smoke.py --managed-spot
+
 # Only run test for AWS + generic tests
 pytest tests/test_smoke.py --aws
 

tests/conftest.py

@@ -13,6 +13,7 @@
 # due to the cloud credit limit for the development account.
 # To only run tests for a specific cloud (as well as generic tests), use
 # --aws, --gcp, or --azure.
+# To only run tests for managed spot (without generic tests), use --managed-spot.
 # A "generic test" tests a generic functionality (e.g., autostop) that
 # should work on any cloud we support. The cloud used for such a test
 # is controlled by `--generic-cloud` (typically you do not need to set it).
@@ -25,12 +26,16 @@ def pytest_addoption(parser):
     parser.addoption('--runslow',
                      action='store_true',
                      default=False,
-                     help='run slow tests')
+                     help='run slow tests.')
     for cloud in all_clouds_in_smoke_tests:
         parser.addoption(f'--{cloud}',
                          action='store_true',
                          default=False,
-                         help=f'Only run {cloud.upper()} tests')
+                         help=f'Only run {cloud.upper()} tests.')
+    parser.addoption('--managed-spot',
+                     action='store_true',
+                     default=False,
+                     help='Only run tests for managed spot.')
     parser.addoption(
         '--generic-cloud',
         type=str,
@@ -59,8 +64,10 @@ def _get_cloud_to_run(config) -> List[str]:
 
 
 def pytest_collection_modifyitems(config, items):
-    skip_slow = pytest.mark.skip(reason='need --runslow option to run')
     skip_marks = {}
+    skip_marks['slow'] = pytest.mark.skip(reason='need --runslow option to run')
+    skip_marks['managed_spot'] = pytest.mark.skip(
+        reason='skipped, because --managed-spot option is set')
     for cloud in all_clouds_in_smoke_tests:
         skip_marks[cloud] = pytest.mark.skip(
             reason=f'tests for {cloud} is skipped, try setting --{cloud}')
@@ -69,11 +76,15 @@ def pytest_collection_modifyitems(config, items):
 
     for item in items:
         if 'slow' in item.keywords and not config.getoption('--runslow'):
-            item.add_marker(skip_slow)
+            item.add_marker(skip_marks['slow'])
         for cloud in all_clouds_in_smoke_tests:
             if cloud in item.keywords and cloud not in cloud_to_run:
                 item.add_marker(skip_marks[cloud])
 
+        if (not 'managed_spot'
+                in item.keywords) and config.getoption('--managed-spot'):
+            item.add_marker(skip_marks['managed_spot'])
+
 
 @pytest.fixture
 def generic_cloud(request) -> str:

tests/test_smoke.py

@@ -1099,6 +1099,7 @@ def test_use_spot(generic_cloud: str):
 
 
 # ---------- Testing managed spot ----------
+@pytest.mark.managed_spot
 def test_spot(generic_cloud: str):
     """Test the spot yaml."""
     name = _get_cluster_name()
@@ -1125,6 +1126,7 @@ def test_spot(generic_cloud: str):
 
 # ---------- Testing managed spot recovery ----------
 @pytest.mark.aws
+@pytest.mark.managed_spot
 def test_spot_recovery_aws():
     """Test managed spot recovery."""
     name = _get_cluster_name()
@@ -1155,6 +1157,7 @@ def test_spot_recovery_aws():
 
 
 @pytest.mark.gcp
+@pytest.mark.managed_spot
 def test_spot_recovery_gcp():
     """Test managed spot recovery."""
     name = _get_cluster_name()
@@ -1185,6 +1188,7 @@ def test_spot_recovery_gcp():
     run_one_test(test)
 
 
+@pytest.mark.managed_spot
 def test_spot_recovery_default_resources(generic_cloud: str):
     """Test managed spot recovery for default resources."""
     name = _get_cluster_name()
@@ -1202,6 +1206,7 @@ def test_spot_recovery_default_resources(generic_cloud: str):
 
 
 @pytest.mark.aws
+@pytest.mark.managed_spot
 def test_spot_recovery_multi_node_aws():
     """Test managed spot recovery."""
     name = _get_cluster_name()
@@ -1233,6 +1238,7 @@ def test_spot_recovery_multi_node_aws():
 
 
 @pytest.mark.gcp
+@pytest.mark.managed_spot
 def test_spot_recovery_multi_node_gcp():
     """Test managed spot recovery."""
     name = _get_cluster_name()
@@ -1266,6 +1272,7 @@ def test_spot_recovery_multi_node_gcp():
 
 
 @pytest.mark.aws
+@pytest.mark.managed_spot
 def test_spot_cancellation_aws():
     name = _get_cluster_name()
     region = 'us-east-2'
@@ -1326,6 +1333,7 @@ def test_spot_cancellation_aws():
 
 
 @pytest.mark.gcp
+@pytest.mark.managed_spot
 def test_spot_cancellation_gcp():
     name = _get_cluster_name()
     zone = 'us-west3-b'
@@ -1381,6 +1389,7 @@ def test_spot_cancellation_gcp():
 
 
 # ---------- Testing storage for managed spot ----------
+@pytest.mark.managed_spot
 def test_spot_storage(generic_cloud: str):
     """Test storage with managed spot"""
     name = _get_cluster_name()
@@ -1408,6 +1417,7 @@ def test_spot_storage(generic_cloud: str):
 
 # ---------- Testing spot TPU ----------
 @pytest.mark.gcp
+@pytest.mark.managed_spot
 def test_spot_tpu():
     """Test managed spot on TPU."""
     name = _get_cluster_name()
@@ -1443,6 +1453,7 @@ def test_inline_env(generic_cloud: str):
 
 
 # ---------- Testing env for spot ----------
+@pytest.mark.managed_spot
 def test_inline_spot_env(generic_cloud: str):
     """Test env"""
     name = _get_cluster_name()