From cbd65d9032199b6a8e62e70228a756e8625104b5 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Wed, 2 Oct 2024 10:32:51 +0100 Subject: [PATCH 1/2] refactor: ray and appwrapper modules Signed-off-by: Bobbins228 --- src/codeflare_sdk/__init__.py | 17 ++- src/codeflare_sdk/cluster/__init__.py | 18 --- src/codeflare_sdk/cluster/widgets.py | 18 +-- .../{ => common}/utils/__init__.py | 0 src/codeflare_sdk/{ => common}/utils/demos.py | 0 .../{ => common}/utils/generate_cert.py | 4 +- src/codeflare_sdk/ray/__init__.py | 16 +++ src/codeflare_sdk/ray/appwrapper/__init__.py | 6 + .../{cluster => ray/appwrapper}/awload.py | 4 +- src/codeflare_sdk/ray/appwrapper/status.py | 46 ++++++++ .../{job => ray/client}/__init__.py | 0 .../{job => ray/client}/ray_jobs.py | 0 src/codeflare_sdk/ray/cluster/__init__.py | 13 +++ .../{ => ray}/cluster/cluster.py | 20 ++-- src/codeflare_sdk/{ => ray}/cluster/config.py | 0 .../{utils => ray/cluster}/generate_yaml.py | 14 +-- .../{utils => ray/cluster}/pretty_print.py | 3 +- .../model.py => ray/cluster/status.py} | 33 +----- .../{ => ray}/templates/base-template.yaml | 0 tests/demo_test.py | 6 +- .../e2e/mnist_raycluster_sdk_aw_kind_test.py | 2 +- tests/e2e/mnist_raycluster_sdk_kind_test.py | 2 +- tests/e2e/mnist_raycluster_sdk_oauth_test.py | 2 +- tests/e2e/mnist_rayjob.py | 4 +- tests/e2e/start_ray_cluster.py | 2 +- tests/unit_test.py | 109 ++++++++++-------- tests/unit_test_support.py | 2 +- .../raycluster_sdk_upgrade_sleep_test.py | 2 +- tests/upgrade/raycluster_sdk_upgrade_test.py | 4 +- 29 files changed, 202 insertions(+), 145 deletions(-) rename src/codeflare_sdk/{ => common}/utils/__init__.py (100%) rename src/codeflare_sdk/{ => common}/utils/demos.py (100%) rename src/codeflare_sdk/{ => common}/utils/generate_cert.py (98%) create mode 100644 src/codeflare_sdk/ray/__init__.py create mode 100644 src/codeflare_sdk/ray/appwrapper/__init__.py rename src/codeflare_sdk/{cluster => ray/appwrapper}/awload.py (97%) create mode 100644 src/codeflare_sdk/ray/appwrapper/status.py rename src/codeflare_sdk/{job => ray/client}/__init__.py (100%) rename src/codeflare_sdk/{job => ray/client}/ray_jobs.py (100%) create mode 100644 src/codeflare_sdk/ray/cluster/__init__.py rename src/codeflare_sdk/{ => ray}/cluster/cluster.py (99%) rename src/codeflare_sdk/{ => ray}/cluster/config.py (100%) rename src/codeflare_sdk/{utils => ray/cluster}/generate_yaml.py (96%) rename src/codeflare_sdk/{utils => ray/cluster}/pretty_print.py (98%) rename src/codeflare_sdk/{cluster/model.py => ray/cluster/status.py} (72%) rename src/codeflare_sdk/{ => ray}/templates/base-template.yaml (100%) diff --git a/src/codeflare_sdk/__init__.py b/src/codeflare_sdk/__init__.py index ba191bc5c..599171a9b 100644 --- a/src/codeflare_sdk/__init__.py +++ b/src/codeflare_sdk/__init__.py @@ -1,18 +1,19 @@ -from .cluster import ( - AWManager, +from .ray import ( Cluster, ClusterConfiguration, RayClusterStatus, - AppWrapperStatus, CodeFlareClusterStatus, RayCluster, - AppWrapper, get_cluster, list_all_queued, list_all_clusters, - view_clusters, + AWManager, + AppWrapperStatus, + RayJobClient, ) +from .cluster import view_clusters + from .common import ( Authentication, KubeConfiguration, @@ -20,10 +21,8 @@ KubeConfigFileAuthentication, ) -from .job import RayJobClient - -from .utils import generate_cert -from .utils.demos import copy_demo_nbs +from .common.utils import generate_cert +from .common.utils.demos import copy_demo_nbs from importlib.metadata import version, PackageNotFoundError diff --git a/src/codeflare_sdk/cluster/__init__.py b/src/codeflare_sdk/cluster/__init__.py index 5be1c9a5a..60be4fcd2 100644 --- a/src/codeflare_sdk/cluster/__init__.py +++ b/src/codeflare_sdk/cluster/__init__.py @@ -1,21 +1,3 @@ -from .model import ( - RayClusterStatus, - AppWrapperStatus, - CodeFlareClusterStatus, - RayCluster, - AppWrapper, -) - -from .cluster import ( - Cluster, - ClusterConfiguration, - get_cluster, - list_all_queued, - list_all_clusters, -) - from .widgets import ( view_clusters, ) - -from .awload import AWManager diff --git a/src/codeflare_sdk/cluster/widgets.py b/src/codeflare_sdk/cluster/widgets.py index 11a92632e..d827c661f 100644 --- a/src/codeflare_sdk/cluster/widgets.py +++ b/src/codeflare_sdk/cluster/widgets.py @@ -26,8 +26,8 @@ import ipywidgets as widgets from IPython.display import display, HTML, Javascript import pandas as pd -from .config import ClusterConfiguration -from .model import RayClusterStatus +from ..ray.cluster.config import ClusterConfiguration +from ..ray.cluster.status import RayClusterStatus from ..common import _kube_api_error_handling from ..common.kubernetes_cluster.auth import ( config_check, @@ -35,7 +35,9 @@ ) -def cluster_up_down_buttons(cluster: "codeflare_sdk.cluster.Cluster") -> widgets.Button: +def cluster_up_down_buttons( + cluster: "codeflare_sdk.ray.cluster.cluster.Cluster", +) -> widgets.Button: """ The cluster_up_down_buttons function returns two button widgets for a create and delete button. The function uses the appwrapper bool to distinguish between resource type for the tool tip. @@ -115,7 +117,7 @@ def view_clusters(namespace: str = None): ) return # Exit function if not in Jupyter Notebook - from .cluster import get_current_namespace + from ..ray.cluster.cluster import get_current_namespace if not namespace: namespace = get_current_namespace() @@ -278,7 +280,7 @@ def _on_ray_dashboard_button_click( """ _on_ray_dashboard_button_click handles the event when the Open Ray Dashboard button is clicked, opening the Ray Dashboard in a new tab """ - from codeflare_sdk.cluster import Cluster + from codeflare_sdk.ray.cluster import Cluster cluster_name = classification_widget.value namespace = ray_clusters_df[ray_clusters_df["Name"] == classification_widget.value][ @@ -309,7 +311,7 @@ def _on_list_jobs_button_click( """ _on_list_jobs_button_click handles the event when the View Jobs button is clicked, opening the Ray Jobs Dashboard in a new tab """ - from codeflare_sdk.cluster import Cluster + from codeflare_sdk.ray.cluster import Cluster cluster_name = classification_widget.value namespace = ray_clusters_df[ray_clusters_df["Name"] == classification_widget.value][ @@ -342,7 +344,7 @@ def _delete_cluster( _delete_cluster function deletes the cluster with the given name and namespace. It optionally waits for the cluster to be deleted. """ - from .cluster import _check_aw_exists + from ..ray.cluster.cluster import _check_aw_exists try: config_check() @@ -400,7 +402,7 @@ def _fetch_cluster_data(namespace): """ _fetch_cluster_data function fetches all clusters and their spec in a given namespace and returns a DataFrame. """ - from .cluster import list_all_clusters + from ..ray.cluster.cluster import list_all_clusters rayclusters = list_all_clusters(namespace, False) if not rayclusters: diff --git a/src/codeflare_sdk/utils/__init__.py b/src/codeflare_sdk/common/utils/__init__.py similarity index 100% rename from src/codeflare_sdk/utils/__init__.py rename to src/codeflare_sdk/common/utils/__init__.py diff --git a/src/codeflare_sdk/utils/demos.py b/src/codeflare_sdk/common/utils/demos.py similarity index 100% rename from src/codeflare_sdk/utils/demos.py rename to src/codeflare_sdk/common/utils/demos.py diff --git a/src/codeflare_sdk/utils/generate_cert.py b/src/codeflare_sdk/common/utils/generate_cert.py similarity index 98% rename from src/codeflare_sdk/utils/generate_cert.py rename to src/codeflare_sdk/common/utils/generate_cert.py index 6d2c0d91f..014d192d7 100644 --- a/src/codeflare_sdk/utils/generate_cert.py +++ b/src/codeflare_sdk/common/utils/generate_cert.py @@ -19,12 +19,12 @@ from cryptography import x509 from cryptography.x509.oid import NameOID import datetime -from ..common.kubernetes_cluster.auth import ( +from ..kubernetes_cluster.auth import ( config_check, get_api_client, ) from kubernetes import client -from ..common import _kube_api_error_handling +from .. import _kube_api_error_handling def generate_ca_cert(days: int = 30): diff --git a/src/codeflare_sdk/ray/__init__.py b/src/codeflare_sdk/ray/__init__.py new file mode 100644 index 000000000..ab55cc824 --- /dev/null +++ b/src/codeflare_sdk/ray/__init__.py @@ -0,0 +1,16 @@ +from .appwrapper import AppWrapper, AppWrapperStatus, AWManager + +from .client import ( + RayJobClient, +) + +from .cluster import ( + Cluster, + ClusterConfiguration, + get_cluster, + list_all_queued, + list_all_clusters, + RayClusterStatus, + CodeFlareClusterStatus, + RayCluster, +) diff --git a/src/codeflare_sdk/ray/appwrapper/__init__.py b/src/codeflare_sdk/ray/appwrapper/__init__.py new file mode 100644 index 000000000..537fdf8aa --- /dev/null +++ b/src/codeflare_sdk/ray/appwrapper/__init__.py @@ -0,0 +1,6 @@ +from .awload import AWManager + +from .status import ( + AppWrapperStatus, + AppWrapper, +) diff --git a/src/codeflare_sdk/cluster/awload.py b/src/codeflare_sdk/ray/appwrapper/awload.py similarity index 97% rename from src/codeflare_sdk/cluster/awload.py rename to src/codeflare_sdk/ray/appwrapper/awload.py index e840c1ca4..02794f3d8 100644 --- a/src/codeflare_sdk/cluster/awload.py +++ b/src/codeflare_sdk/ray/appwrapper/awload.py @@ -23,8 +23,8 @@ import yaml from kubernetes import client -from ..common import _kube_api_error_handling -from ..common.kubernetes_cluster.auth import ( +from ...common import _kube_api_error_handling +from ...common.kubernetes_cluster.auth import ( config_check, get_api_client, ) diff --git a/src/codeflare_sdk/ray/appwrapper/status.py b/src/codeflare_sdk/ray/appwrapper/status.py new file mode 100644 index 000000000..79fe0fd2a --- /dev/null +++ b/src/codeflare_sdk/ray/appwrapper/status.py @@ -0,0 +1,46 @@ +# Copyright 2024 IBM, Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +The status sub-module defines Enums containing information for +AppWrapper states, as well as dataclasses to store information for AppWrappers. +""" + +from dataclasses import dataclass +from enum import Enum + + +class AppWrapperStatus(Enum): + """ + Defines the possible reportable phases of an AppWrapper. + """ + + SUSPENDED = "suspended" + RESUMING = "resuming" + RUNNING = "running" + RESETTING = "resetting" + SUSPENDING = "suspending" + SUCCEEDED = "succeeded" + FAILED = "failed" + TERMINATING = "terminating" + + +@dataclass +class AppWrapper: + """ + For storing information about an AppWrapper. + """ + + name: str + status: AppWrapperStatus diff --git a/src/codeflare_sdk/job/__init__.py b/src/codeflare_sdk/ray/client/__init__.py similarity index 100% rename from src/codeflare_sdk/job/__init__.py rename to src/codeflare_sdk/ray/client/__init__.py diff --git a/src/codeflare_sdk/job/ray_jobs.py b/src/codeflare_sdk/ray/client/ray_jobs.py similarity index 100% rename from src/codeflare_sdk/job/ray_jobs.py rename to src/codeflare_sdk/ray/client/ray_jobs.py diff --git a/src/codeflare_sdk/ray/cluster/__init__.py b/src/codeflare_sdk/ray/cluster/__init__.py new file mode 100644 index 000000000..bf32459be --- /dev/null +++ b/src/codeflare_sdk/ray/cluster/__init__.py @@ -0,0 +1,13 @@ +from .status import ( + RayClusterStatus, + CodeFlareClusterStatus, + RayCluster, +) + +from .cluster import ( + Cluster, + ClusterConfiguration, + get_cluster, + list_all_queued, + list_all_clusters, +) diff --git a/src/codeflare_sdk/cluster/cluster.py b/src/codeflare_sdk/ray/cluster/cluster.py similarity index 99% rename from src/codeflare_sdk/cluster/cluster.py rename to src/codeflare_sdk/ray/cluster/cluster.py index 773082451..4d8201f78 100644 --- a/src/codeflare_sdk/cluster/cluster.py +++ b/src/codeflare_sdk/ray/cluster/cluster.py @@ -23,27 +23,29 @@ from ray.job_submission import JobSubmissionClient -from ..common.kubernetes_cluster.auth import ( +from ...common.kubernetes_cluster.auth import ( config_check, get_api_client, ) -from ..utils import pretty_print -from ..utils.generate_yaml import ( +from . import pretty_print +from .generate_yaml import ( generate_appwrapper, head_worker_gpu_count_from_cluster, ) -from ..common import _kube_api_error_handling -from ..utils.generate_yaml import is_openshift_cluster +from ...common import _kube_api_error_handling +from .generate_yaml import is_openshift_cluster from .config import ClusterConfiguration -from .model import ( - AppWrapper, - AppWrapperStatus, +from .status import ( CodeFlareClusterStatus, RayCluster, RayClusterStatus, ) -from .widgets import ( +from ..appwrapper import ( + AppWrapper, + AppWrapperStatus, +) +from ...cluster.widgets import ( cluster_up_down_buttons, is_notebook, ) diff --git a/src/codeflare_sdk/cluster/config.py b/src/codeflare_sdk/ray/cluster/config.py similarity index 100% rename from src/codeflare_sdk/cluster/config.py rename to src/codeflare_sdk/ray/cluster/config.py diff --git a/src/codeflare_sdk/utils/generate_yaml.py b/src/codeflare_sdk/ray/cluster/generate_yaml.py similarity index 96% rename from src/codeflare_sdk/utils/generate_yaml.py rename to src/codeflare_sdk/ray/cluster/generate_yaml.py index 2ddfb31ac..f0d70cf52 100755 --- a/src/codeflare_sdk/utils/generate_yaml.py +++ b/src/codeflare_sdk/ray/cluster/generate_yaml.py @@ -24,8 +24,8 @@ import os import uuid from kubernetes import client -from ..common import _kube_api_error_handling -from ..common.kubernetes_cluster.auth import ( +from ...common import _kube_api_error_handling +from ...common.kubernetes_cluster.auth import ( get_api_client, config_check, ) @@ -80,7 +80,7 @@ def is_kind_cluster(): def update_names( cluster_yaml: dict, - cluster: "codeflare_sdk.cluster.Cluster", + cluster: "codeflare_sdk.ray.cluster.cluster.Cluster", ): metadata = cluster_yaml.get("metadata") metadata["name"] = cluster.config.name @@ -135,7 +135,7 @@ def update_resources( def head_worker_gpu_count_from_cluster( - cluster: "codeflare_sdk.cluster.Cluster", + cluster: "codeflare_sdk.ray.cluster.cluster.Cluster", ) -> typing.Tuple[int, int]: head_gpus = 0 worker_gpus = 0 @@ -155,7 +155,7 @@ def head_worker_gpu_count_from_cluster( def head_worker_resources_from_cluster( - cluster: "codeflare_sdk.cluster.Cluster", + cluster: "codeflare_sdk.ray.cluster.cluster.Cluster", ) -> typing.Tuple[dict, dict]: to_return = {}, {} for k in cluster.config.head_extended_resource_requests.keys(): @@ -178,7 +178,7 @@ def head_worker_resources_from_cluster( def update_nodes( ray_cluster_dict: dict, - cluster: "codeflare_sdk.cluster.Cluster", + cluster: "codeflare_sdk.ray.cluster.cluster.Cluster", ): head = ray_cluster_dict.get("spec").get("headGroupSpec") worker = ray_cluster_dict.get("spec").get("workerGroupSpecs")[0] @@ -325,7 +325,7 @@ def write_user_yaml(user_yaml, output_file_name): print(f"Written to: {output_file_name}") -def generate_appwrapper(cluster: "codeflare_sdk.cluster.Cluster"): +def generate_appwrapper(cluster: "codeflare_sdk.ray.cluster.cluster.Cluster"): cluster_yaml = read_template(cluster.config.template) appwrapper_name, _ = gen_names(cluster.config.name) update_names( diff --git a/src/codeflare_sdk/utils/pretty_print.py b/src/codeflare_sdk/ray/cluster/pretty_print.py similarity index 98% rename from src/codeflare_sdk/utils/pretty_print.py rename to src/codeflare_sdk/ray/cluster/pretty_print.py index 303313199..883f14adc 100644 --- a/src/codeflare_sdk/utils/pretty_print.py +++ b/src/codeflare_sdk/ray/cluster/pretty_print.py @@ -24,7 +24,8 @@ from rich.panel import Panel from rich import box from typing import List -from ..cluster.model import RayCluster, AppWrapper, RayClusterStatus +from .status import RayCluster, RayClusterStatus +from ..appwrapper.status import AppWrapper def print_no_resources_found(): diff --git a/src/codeflare_sdk/cluster/model.py b/src/codeflare_sdk/ray/cluster/status.py similarity index 72% rename from src/codeflare_sdk/cluster/model.py rename to src/codeflare_sdk/ray/cluster/status.py index 44be54567..136ae302d 100644 --- a/src/codeflare_sdk/cluster/model.py +++ b/src/codeflare_sdk/ray/cluster/status.py @@ -1,4 +1,4 @@ -# Copyright 2022 IBM, Red Hat +# Copyright 2024 IBM, Red Hat # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,9 +13,9 @@ # limitations under the License. """ -The model sub-module defines Enums containing information for Ray cluster -states and AppWrapper states, and CodeFlare cluster states, as well as -dataclasses to store information for Ray clusters and AppWrappers. +The status sub-module defines Enums containing information for Ray cluster +states states, and CodeFlare cluster states, as well as +dataclasses to store information for Ray clusters. """ from dataclasses import dataclass, field @@ -37,21 +37,6 @@ class RayClusterStatus(Enum): SUSPENDED = "suspended" -class AppWrapperStatus(Enum): - """ - Defines the possible reportable phases of an AppWrapper. - """ - - SUSPENDED = "suspended" - RESUMING = "resuming" - RUNNING = "running" - RESETTING = "resetting" - SUSPENDING = "suspending" - SUCCEEDED = "succeeded" - FAILED = "failed" - TERMINATING = "terminating" - - class CodeFlareClusterStatus(Enum): """ Defines the possible reportable states of a Codeflare cluster. @@ -87,13 +72,3 @@ class RayCluster: dashboard: str worker_extended_resources: typing.Dict[str, int] = field(default_factory=dict) head_extended_resources: typing.Dict[str, int] = field(default_factory=dict) - - -@dataclass -class AppWrapper: - """ - For storing information about an AppWrapper. - """ - - name: str - status: AppWrapperStatus diff --git a/src/codeflare_sdk/templates/base-template.yaml b/src/codeflare_sdk/ray/templates/base-template.yaml similarity index 100% rename from src/codeflare_sdk/templates/base-template.yaml rename to src/codeflare_sdk/ray/templates/base-template.yaml diff --git a/tests/demo_test.py b/tests/demo_test.py index 65324af08..b54530580 100644 --- a/tests/demo_test.py +++ b/tests/demo_test.py @@ -16,12 +16,12 @@ These were the old tests used during initial demo building, and they will soon be fully deprecated. """ -from codeflare_sdk.cluster.cluster import ( +from codeflare_sdk.ray.cluster.cluster import ( list_all_clusters, list_all_queued, _app_wrapper_status, ) -from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration +from codeflare_sdk.ray.cluster.cluster import Cluster, ClusterConfiguration import time @@ -53,7 +53,7 @@ def test_cluster_down(): def test_no_resources_found(): - from codeflare_sdk.utils import pretty_print + from codeflare_sdk.ray.cluster import pretty_print pretty_print.print_no_resources_found() diff --git a/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py b/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py index d7949b8cb..4623a9e53 100644 --- a/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py +++ b/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py @@ -3,7 +3,7 @@ from time import sleep from codeflare_sdk import Cluster, ClusterConfiguration, TokenAuthentication -from codeflare_sdk.job import RayJobClient +from codeflare_sdk.ray.client import RayJobClient import pytest diff --git a/tests/e2e/mnist_raycluster_sdk_kind_test.py b/tests/e2e/mnist_raycluster_sdk_kind_test.py index d81e9149b..6bfb19af0 100644 --- a/tests/e2e/mnist_raycluster_sdk_kind_test.py +++ b/tests/e2e/mnist_raycluster_sdk_kind_test.py @@ -3,7 +3,7 @@ from time import sleep from codeflare_sdk import Cluster, ClusterConfiguration, TokenAuthentication -from codeflare_sdk.job import RayJobClient +from codeflare_sdk.ray.client import RayJobClient import pytest diff --git a/tests/e2e/mnist_raycluster_sdk_oauth_test.py b/tests/e2e/mnist_raycluster_sdk_oauth_test.py index 7e4002adf..d3e698682 100644 --- a/tests/e2e/mnist_raycluster_sdk_oauth_test.py +++ b/tests/e2e/mnist_raycluster_sdk_oauth_test.py @@ -3,7 +3,7 @@ from time import sleep from codeflare_sdk import Cluster, ClusterConfiguration, TokenAuthentication -from codeflare_sdk.job import RayJobClient +from codeflare_sdk.ray.client import RayJobClient import pytest diff --git a/tests/e2e/mnist_rayjob.py b/tests/e2e/mnist_rayjob.py index c9306da69..bf47e6da6 100644 --- a/tests/e2e/mnist_rayjob.py +++ b/tests/e2e/mnist_rayjob.py @@ -4,8 +4,8 @@ from support import * -from codeflare_sdk.cluster.cluster import get_cluster -from codeflare_sdk.job import RayJobClient +from codeflare_sdk.ray.cluster.cluster import get_cluster +from codeflare_sdk.ray.client import RayJobClient namespace = sys.argv[1] diff --git a/tests/e2e/start_ray_cluster.py b/tests/e2e/start_ray_cluster.py index 48ab604ff..8aac19f09 100644 --- a/tests/e2e/start_ray_cluster.py +++ b/tests/e2e/start_ray_cluster.py @@ -3,7 +3,7 @@ from time import sleep -from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration +from codeflare_sdk.ray.cluster.cluster import Cluster, ClusterConfiguration namespace = sys.argv[1] ray_image = os.getenv("RAY_IMAGE") diff --git a/tests/unit_test.py b/tests/unit_test.py index bb2e2f7d6..235eed0e7 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -20,15 +20,15 @@ import re import uuid -from codeflare_sdk.cluster import cluster +from codeflare_sdk.ray.cluster import cluster parent = Path(__file__).resolve().parents[1] aw_dir = os.path.expanduser("~/.codeflare/resources/") sys.path.append(str(parent) + "/src") from kubernetes import client, config, dynamic -from codeflare_sdk.cluster.awload import AWManager -from codeflare_sdk.cluster.cluster import ( +from codeflare_sdk.ray.appwrapper.awload import AWManager +from codeflare_sdk.ray.cluster.cluster import ( Cluster, ClusterConfiguration, _map_to_ray_cluster, @@ -45,20 +45,22 @@ KubeConfigFileAuthentication, config_check, ) -from codeflare_sdk.utils.pretty_print import ( +from codeflare_sdk.ray.cluster.pretty_print import ( print_no_resources_found, print_app_wrappers_status, print_cluster_status, print_clusters, ) -from codeflare_sdk.cluster.model import ( +from codeflare_sdk.ray.appwrapper.status import ( AppWrapper, - RayCluster, AppWrapperStatus, +) +from codeflare_sdk.ray.cluster.status import ( + RayCluster, RayClusterStatus, CodeFlareClusterStatus, ) -from codeflare_sdk.utils.generate_cert import ( +from codeflare_sdk.common.utils.generate_cert import ( generate_ca_cert, generate_tls_cert, export_env, @@ -72,7 +74,7 @@ ) import codeflare_sdk.common.kubernetes_cluster.kube_api_helpers -from codeflare_sdk.utils.generate_yaml import ( +from codeflare_sdk.ray.cluster.generate_yaml import ( gen_names, is_openshift_cluster, ) @@ -88,7 +90,7 @@ from unittest.mock import MagicMock, patch from pytest_mock import MockerFixture from ray.job_submission import JobSubmissionClient -from codeflare_sdk.job.ray_jobs import RayJobClient +from codeflare_sdk.ray.client.ray_jobs import RayJobClient # For mocking openshift client results fake_res = openshift.Result("fake") @@ -268,7 +270,10 @@ def test_config_creation(): assert config.worker_cpu_requests == 3 and config.worker_cpu_limits == 4 assert config.worker_memory_requests == "5G" and config.worker_memory_limits == "6G" assert config.worker_extended_resource_requests == {"nvidia.com/gpu": 7} - assert config.template == f"{parent}/src/codeflare_sdk/templates/base-template.yaml" + assert ( + config.template + == f"{parent}/src/codeflare_sdk/ray/templates/base-template.yaml" + ) assert config.machine_types == ["cpu.small", "gpu.large"] assert config.image_pull_secrets == ["unit-test-pull-secret"] assert config.appwrapper == True @@ -438,7 +443,7 @@ def test_cluster_creation_no_mcad_local_queue(mocker): def test_default_cluster_creation(mocker): mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch( - "codeflare_sdk.cluster.cluster.get_current_namespace", + "codeflare_sdk.ray.cluster.cluster.get_current_namespace", return_value="opendatahub", ) mocker.patch( @@ -593,7 +598,7 @@ def arg_check_del_effect(group, version, namespace, plural, name, *args): def test_cluster_up_down(mocker): mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") - mocker.patch("codeflare_sdk.cluster.cluster.Cluster._throw_for_no_raycluster") + mocker.patch("codeflare_sdk.ray.cluster.cluster.Cluster._throw_for_no_raycluster") mocker.patch( "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", return_value={"spec": {"domain": ""}}, @@ -620,7 +625,7 @@ def test_cluster_up_down(mocker): def test_cluster_up_down_no_mcad(mocker): - mocker.patch("codeflare_sdk.cluster.cluster.Cluster._throw_for_no_raycluster") + mocker.patch("codeflare_sdk.ray.cluster.cluster.Cluster._throw_for_no_raycluster") mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch( @@ -716,7 +721,7 @@ def test_cluster_uris(mocker): mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") mocker.patch( - "codeflare_sdk.cluster.cluster._get_ingress_domain", + "codeflare_sdk.ray.cluster.cluster._get_ingress_domain", return_value="apps.cluster.awsroute.org", ) mocker.patch( @@ -760,11 +765,11 @@ def test_local_client_url(mocker): return_value={"spec": {"domain": ""}}, ) mocker.patch( - "codeflare_sdk.cluster.cluster._get_ingress_domain", + "codeflare_sdk.ray.cluster.cluster._get_ingress_domain", return_value="rayclient-unit-test-cluster-localinter-ns.apps.cluster.awsroute.org", ) mocker.patch( - "codeflare_sdk.cluster.cluster.Cluster.create_app_wrapper", + "codeflare_sdk.ray.cluster.cluster.Cluster.create_app_wrapper", return_value="unit-test-cluster-localinter.yaml", ) @@ -955,15 +960,15 @@ def test_ray_details(mocker, capsys): head_mem_limits=8, ) mocker.patch( - "codeflare_sdk.cluster.cluster.Cluster.status", + "codeflare_sdk.ray.cluster.cluster.Cluster.status", return_value=(False, CodeFlareClusterStatus.UNKNOWN), ) mocker.patch( - "codeflare_sdk.cluster.cluster.Cluster.cluster_dashboard_uri", + "codeflare_sdk.ray.cluster.cluster.Cluster.cluster_dashboard_uri", return_value="", ) mocker.patch( - "codeflare_sdk.utils.generate_yaml.local_queue_exists", + "codeflare_sdk.ray.cluster.generate_yaml.local_queue_exists", return_value="true", ) cf = Cluster( @@ -2002,7 +2007,7 @@ def test_get_cluster_openshift(mocker): ] mocker.patch("kubernetes.client.ApisApi", return_value=mock_api) mocker.patch( - "codeflare_sdk.utils.generate_yaml.local_queue_exists", + "codeflare_sdk.ray.cluster.generate_yaml.local_queue_exists", return_value="true", ) @@ -2037,7 +2042,7 @@ def custom_side_effect(group, version, namespace, plural, **kwargs): ], ) mocker.patch( - "codeflare_sdk.utils.generate_yaml.local_queue_exists", + "codeflare_sdk.ray.cluster.generate_yaml.local_queue_exists", return_value="true", ) @@ -2080,7 +2085,7 @@ def test_get_cluster(mocker): return_value=ingress_retrieval(cluster_name="quicktest", client_ing=True), ) mocker.patch( - "codeflare_sdk.utils.generate_yaml.local_queue_exists", + "codeflare_sdk.ray.cluster.generate_yaml.local_queue_exists", return_value="true", ) cluster = get_cluster("quicktest") @@ -2118,7 +2123,7 @@ def test_get_cluster_no_mcad(mocker): return_value=ingress_retrieval(cluster_name="quicktest", client_ing=True), ) mocker.patch( - "codeflare_sdk.utils.generate_yaml.local_queue_exists", + "codeflare_sdk.ray.cluster.generate_yaml.local_queue_exists", return_value="true", ) cluster = get_cluster("quicktest") @@ -2167,7 +2172,7 @@ def test_map_to_ray_cluster(mocker): mocker.patch("kubernetes.config.load_kube_config") mocker.patch( - "codeflare_sdk.cluster.cluster.is_openshift_cluster", return_value=True + "codeflare_sdk.ray.cluster.cluster.is_openshift_cluster", return_value=True ) mock_api_client = mocker.MagicMock(spec=client.ApiClient) @@ -2354,7 +2359,7 @@ def test_cluster_status(mocker): mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") mocker.patch( - "codeflare_sdk.utils.generate_yaml.local_queue_exists", + "codeflare_sdk.ray.cluster.generate_yaml.local_queue_exists", return_value="true", ) fake_aw = AppWrapper("test", AppWrapperStatus.FAILED) @@ -2382,14 +2387,18 @@ def test_cluster_status(mocker): local_queue="local_default_queue", ) ) - mocker.patch("codeflare_sdk.cluster.cluster._app_wrapper_status", return_value=None) - mocker.patch("codeflare_sdk.cluster.cluster._ray_cluster_status", return_value=None) + mocker.patch( + "codeflare_sdk.ray.cluster.cluster._app_wrapper_status", return_value=None + ) + mocker.patch( + "codeflare_sdk.ray.cluster.cluster._ray_cluster_status", return_value=None + ) status, ready = cf.status() assert status == CodeFlareClusterStatus.UNKNOWN assert ready == False mocker.patch( - "codeflare_sdk.cluster.cluster._app_wrapper_status", return_value=fake_aw + "codeflare_sdk.ray.cluster.cluster._app_wrapper_status", return_value=fake_aw ) status, ready = cf.status() assert status == CodeFlareClusterStatus.FAILED @@ -2416,7 +2425,7 @@ def test_cluster_status(mocker): assert ready == False mocker.patch( - "codeflare_sdk.cluster.cluster._ray_cluster_status", return_value=fake_ray + "codeflare_sdk.ray.cluster.cluster._ray_cluster_status", return_value=fake_ray ) status, ready = cf.status() @@ -2446,10 +2455,14 @@ def test_wait_ready(mocker, capsys): return_value=ingress_retrieval(), ) mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") - mocker.patch("codeflare_sdk.cluster.cluster._app_wrapper_status", return_value=None) - mocker.patch("codeflare_sdk.cluster.cluster._ray_cluster_status", return_value=None) mocker.patch( - "codeflare_sdk.utils.generate_yaml.local_queue_exists", + "codeflare_sdk.ray.cluster.cluster._app_wrapper_status", return_value=None + ) + mocker.patch( + "codeflare_sdk.ray.cluster.cluster._ray_cluster_status", return_value=None + ) + mocker.patch( + "codeflare_sdk.ray.cluster.generate_yaml.local_queue_exists", return_value="true", ) mocker.patch.object( @@ -2488,7 +2501,7 @@ def test_wait_ready(mocker, capsys): in captured.out ) mocker.patch( - "codeflare_sdk.cluster.cluster.Cluster.status", + "codeflare_sdk.ray.cluster.cluster.Cluster.status", return_value=(True, CodeFlareClusterStatus.READY), ) cf.wait_ready() @@ -2599,7 +2612,7 @@ def test_AWManager_submit_remove(mocker, capsys): def test_generate_ca_cert(): """ - test the function codeflare_sdk.utils.generate_ca_cert generates the correct outputs + test the function codeflare_sdk.common.utils.generate_ca_cert generates the correct outputs """ key, certificate = generate_ca_cert() cert = load_pem_x509_certificate(base64.b64decode(certificate)) @@ -2629,11 +2642,11 @@ def secret_ca_retreival(secret_name, namespace): def test_generate_tls_cert(mocker): """ - test the function codeflare_sdk.utils.generate_ca_cert generates the correct outputs + test the function codeflare_sdk.common.utils.generate_ca_cert generates the correct outputs """ mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") mocker.patch( - "codeflare_sdk.utils.generate_cert.get_secret_name", + "codeflare_sdk.common.utils.generate_cert.get_secret_name", return_value="ca-secret-cluster", ) mocker.patch( @@ -2657,7 +2670,7 @@ def test_generate_tls_cert(mocker): def test_export_env(): """ - test the function codeflare_sdk.utils.export_ev generates the correct outputs + test the function codeflare_sdk.common.utils.generate_ca_cert.export_ev generates the correct outputs """ tls_dir = "cluster" ns = "namespace" @@ -2677,15 +2690,15 @@ def test_export_env(): def test_cluster_throw_for_no_raycluster(mocker: MockerFixture): mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch( - "codeflare_sdk.cluster.cluster.get_current_namespace", + "codeflare_sdk.ray.cluster.cluster.get_current_namespace", return_value="opendatahub", ) mocker.patch( - "codeflare_sdk.utils.generate_yaml.get_default_kueue_name", + "codeflare_sdk.ray.cluster.generate_yaml.get_default_kueue_name", return_value="default", ) mocker.patch( - "codeflare_sdk.utils.generate_yaml.local_queue_exists", + "codeflare_sdk.ray.cluster.generate_yaml.local_queue_exists", return_value="true", ) @@ -2980,12 +2993,14 @@ def test_view_clusters(mocker, capsys): "kubernetes.client.CustomObjectsApi.list_namespaced_custom_object", return_value={"items": []}, ) - mocker.patch("codeflare_sdk.cluster.cluster._check_aw_exists", return_value=False) + mocker.patch( + "codeflare_sdk.ray.cluster.cluster._check_aw_exists", return_value=False + ) # Return empty dataframe when no clusters are found - mocker.patch("codeflare_sdk.cluster.cluster.list_all_clusters", return_value=[]) + mocker.patch("codeflare_sdk.ray.cluster.cluster.list_all_clusters", return_value=[]) mocker.patch( - "codeflare_sdk.cluster.cluster.get_current_namespace", + "codeflare_sdk.ray.cluster.cluster.get_current_namespace", return_value="default", ) df = cf_widgets._fetch_cluster_data(namespace="default") @@ -3019,8 +3034,8 @@ def test_view_clusters(mocker, capsys): ) # Mock the Cluster class and related methods - mocker.patch("codeflare_sdk.cluster.Cluster") - mocker.patch("codeflare_sdk.cluster.ClusterConfiguration") + mocker.patch("codeflare_sdk.ray.cluster.Cluster") + mocker.patch("codeflare_sdk.ray.cluster.ClusterConfiguration") with patch("ipywidgets.ToggleButtons") as MockToggleButtons, patch( "ipywidgets.Button" @@ -3103,7 +3118,7 @@ def test_view_clusters(mocker, capsys): def test_fetch_cluster_data(mocker): # Return empty dataframe when no clusters are found - mocker.patch("codeflare_sdk.cluster.cluster.list_all_clusters", return_value=[]) + mocker.patch("codeflare_sdk.ray.cluster.cluster.list_all_clusters", return_value=[]) df = cf_widgets._fetch_cluster_data(namespace="default") assert df.empty @@ -3145,7 +3160,7 @@ def test_fetch_cluster_data(mocker): mock_raycluster2.status = RayClusterStatus.SUSPENDED with patch( - "codeflare_sdk.cluster.cluster.list_all_clusters", + "codeflare_sdk.ray.cluster.cluster.list_all_clusters", return_value=[mock_raycluster1, mock_raycluster2], ): # Call the function under test diff --git a/tests/unit_test_support.py b/tests/unit_test_support.py index c23c47ebf..b3c2e1977 100644 --- a/tests/unit_test_support.py +++ b/tests/unit_test_support.py @@ -1,4 +1,4 @@ -from codeflare_sdk.cluster.cluster import ( +from codeflare_sdk.ray.cluster.cluster import ( Cluster, ClusterConfiguration, ) diff --git a/tests/upgrade/raycluster_sdk_upgrade_sleep_test.py b/tests/upgrade/raycluster_sdk_upgrade_sleep_test.py index 65c836ed1..793853d00 100644 --- a/tests/upgrade/raycluster_sdk_upgrade_sleep_test.py +++ b/tests/upgrade/raycluster_sdk_upgrade_sleep_test.py @@ -7,7 +7,7 @@ TokenAuthentication, get_cluster, ) -from codeflare_sdk.job import RayJobClient +from codeflare_sdk.ray.client import RayJobClient from tests.e2e.support import * diff --git a/tests/upgrade/raycluster_sdk_upgrade_test.py b/tests/upgrade/raycluster_sdk_upgrade_test.py index dce1ca717..7c8b2922e 100644 --- a/tests/upgrade/raycluster_sdk_upgrade_test.py +++ b/tests/upgrade/raycluster_sdk_upgrade_test.py @@ -2,10 +2,10 @@ from time import sleep from codeflare_sdk import Cluster, ClusterConfiguration, TokenAuthentication -from codeflare_sdk.job import RayJobClient +from codeflare_sdk.ray.client import RayJobClient from tests.e2e.support import * -from codeflare_sdk.cluster.cluster import get_cluster +from codeflare_sdk.ray.cluster.cluster import get_cluster from codeflare_sdk.common import _kube_api_error_handling From be8582a64f345b6200564c5915eaa08eb2fccaed Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Wed, 2 Oct 2024 10:57:38 +0100 Subject: [PATCH 2/2] remove codeflare_sdk.egg-info Signed-off-by: Bobbins228 --- src/codeflare_sdk.egg-info/PKG-INFO | 4 --- src/codeflare_sdk.egg-info/SOURCES.txt | 25 ------------------- .../dependency_links.txt | 0 src/codeflare_sdk.egg-info/top_level.txt | 1 - 4 files changed, 30 deletions(-) delete mode 100644 src/codeflare_sdk.egg-info/PKG-INFO delete mode 100644 src/codeflare_sdk.egg-info/SOURCES.txt delete mode 100644 src/codeflare_sdk.egg-info/dependency_links.txt delete mode 100644 src/codeflare_sdk.egg-info/top_level.txt diff --git a/src/codeflare_sdk.egg-info/PKG-INFO b/src/codeflare_sdk.egg-info/PKG-INFO deleted file mode 100644 index c4061c623..000000000 --- a/src/codeflare_sdk.egg-info/PKG-INFO +++ /dev/null @@ -1,4 +0,0 @@ -Metadata-Version: 2.1 -Name: codeflare-sdk -Version: 0.0.0 -License-File: LICENSE diff --git a/src/codeflare_sdk.egg-info/SOURCES.txt b/src/codeflare_sdk.egg-info/SOURCES.txt deleted file mode 100644 index 53043edf1..000000000 --- a/src/codeflare_sdk.egg-info/SOURCES.txt +++ /dev/null @@ -1,25 +0,0 @@ -LICENSE -README.md -pyproject.toml -src/codeflare_sdk/__init__.py -src/codeflare_sdk.egg-info/PKG-INFO -src/codeflare_sdk.egg-info/SOURCES.txt -src/codeflare_sdk.egg-info/dependency_links.txt -src/codeflare_sdk.egg-info/top_level.txt -src/codeflare_sdk/cluster/__init__.py -src/codeflare_sdk/cluster/awload.py -src/codeflare_sdk/cluster/cluster.py -src/codeflare_sdk/cluster/config.py -src/codeflare_sdk/cluster/model.py -src/codeflare_sdk/cluster/widgets.py -src/codeflare_sdk/common/__init__.py -src/codeflare_sdk/common/kubernetes_cluster/__init__.py -src/codeflare_sdk/common/kubernetes_cluster/auth.py -src/codeflare_sdk/common/kubernetes_cluster/kube_api_helpers.py -src/codeflare_sdk/job/__init__.py -src/codeflare_sdk/job/ray_jobs.py -src/codeflare_sdk/utils/__init__.py -src/codeflare_sdk/utils/demos.py -src/codeflare_sdk/utils/generate_cert.py -src/codeflare_sdk/utils/generate_yaml.py -src/codeflare_sdk/utils/pretty_print.py diff --git a/src/codeflare_sdk.egg-info/dependency_links.txt b/src/codeflare_sdk.egg-info/dependency_links.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/codeflare_sdk.egg-info/top_level.txt b/src/codeflare_sdk.egg-info/top_level.txt deleted file mode 100644 index 633675b3c..000000000 --- a/src/codeflare_sdk.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -codeflare_sdk