Fixes ray dashboard hanging problem (skypilot-org#1088) (skypilot-org#1109)

* patch job_manager

* fix cloud API

Author: Michaelvll

sky/clouds/azure.py

@@ -77,10 +77,7 @@ def is_same_cloud(self, other):
         return isinstance(other, Azure)
 
     @classmethod
-    def get_default_instance_type(cls,
-                                  accelerators: Optional[Dict[str, int]] = None
-                                 ) -> str:
-        del accelerators
+    def get_default_instance_type(cls) -> str:
         # 8 vCpus, 32 GB RAM.  Prev-gen (as of 2021) general purpose.
         return 'Standard_D8_v4'
 

sky/clouds/cloud.py

@@ -149,9 +149,7 @@ def get_accelerators_from_instance_type(
         raise NotImplementedError
 
     @classmethod
-    def get_default_instance_type(cls,
-                                  accelerators: Optional[Dict[str, int]] = None
-                                 ) -> str:
+    def get_default_instance_type(cls) -> str:
         raise NotImplementedError
 
     @classmethod

sky/skylet/ray_patches/__init__.py

@@ -66,6 +66,9 @@ def patch() -> None:
     from ray.dashboard.modules.job import cli
     _run_patch(cli.__file__, _to_absolute('cli.py.patch'))
 
+    from ray.dashboard.modules.job import job_manager
+    _run_patch(job_manager.__file__, _to_absolute('job_manager.py.patch'))
+
     from ray.autoscaler._private import autoscaler
     _run_patch(autoscaler.__file__, _to_absolute('autoscaler.py.patch'))
 

sky/skylet/ray_patches/job_manager.py.patch

@@ -0,0 +1,9 @@
+0a1,4
+> # Adapted from https://github.com/ray-project/ray/blob/ray-1.13.0/dashboard/modules/job/job_manager.py
+> # Fixed the problem where the _monitor_job thread is leaked, due to `await job_supervisor.ping.remote()`
+> # does not raise an exception after the job_supervisor is exited, causing the dashboard to hang.
+> 
+334c338
+<                 await job_supervisor.ping.remote()
+---
+>                 ray.get(job_supervisor.ping.remote())