Skip to content

Commit d7ae866

Browse files
MaxusmusticarsonmhtedhtchangBobbins228
authored
K8s Support for SDK (#146)
* First pass on just the basics * Added namespace retrieval and dashboard route access via kubernetes * Added exception handling * Remove unnecessary comment * Change AW fs loading to k8s and begin converting unit tests * Finished unit test update * Update requirements * Get cluster (#189) * Add: get_cluster function to get cluster with specified name and namespace * Test: make unit tests for get_cluster function * Fix: unit test failing because of ray cluster obj changed (#208) * Remove oc client and add helper functions (#187) * Remove oc client and add helper functions * Updates for error checking * make sure tests run without oc login * Removing CLI appwrapper generation * Updated import --------- Co-authored-by: Mustafa Eyceoz <[email protected]> * Remove unused import * Update authentication for K8s (#237) * Updated authentication for Kubernetes * Updated template name and comment * Updated login functionality * Altered config_check() function * Altered comments and changed config_check() function * Added logic for handling current namespace when a user authenticates via kube client * Changed formatting * Made handler functions generic and altered get_current_namespace() functionality * Changed error message for cluster configuration * Removed default values for token + server * Added check for correct credentials * Changed how using certs works with certifi.where * Added unit tests for new authentication methods * Fixed formatting and updated .gitignore to include test created files * Fixed .gitignore * Updated unit authentication tests --------- Co-authored-by: Carson Harrell <[email protected]> Co-authored-by: ted chang <[email protected]> Co-authored-by: Mark Campbell <[email protected]>
1 parent 30ec84c commit d7ae866

File tree

14 files changed

+1525
-1516
lines changed

14 files changed

+1525
-1516
lines changed

Diff for: ‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,5 @@ Pipfile.lock
77
poetry.lock
88
.venv*
99
build/
10+
tls-cluster-namespace
11+
quicktest.yaml

Diff for: ‎demo-notebooks/interactive/local_interactive.ipynb

+31-57
Original file line numberDiff line numberDiff line change
@@ -32,20 +32,12 @@
3232
},
3333
{
3434
"cell_type": "code",
35-
"execution_count": 2,
35+
"execution_count": null,
3636
"id": "4364ac2e-dd10-4d30-ba66-12708daefb3f",
3737
"metadata": {
3838
"tags": []
3939
},
40-
"outputs": [
41-
{
42-
"name": "stdout",
43-
"output_type": "stream",
44-
"text": [
45-
"Written to: hfgputest-1.yaml\n"
46-
]
47-
}
48-
],
40+
"outputs": [],
4941
"source": [
5042
"# Create our cluster and submit appwrapper\n",
5143
"namespace = \"default\"\n",
@@ -89,7 +81,6 @@
8981
]
9082
},
9183
{
92-
"attachments": {},
9384
"cell_type": "markdown",
9485
"id": "12eef53c",
9586
"metadata": {},
@@ -99,38 +90,21 @@
9990
},
10091
{
10192
"cell_type": "code",
102-
"execution_count": 11,
93+
"execution_count": null,
10394
"id": "cf1b749e-2335-42c2-b673-26768ec9895d",
10495
"metadata": {},
105-
"outputs": [
106-
{
107-
"name": "stdout",
108-
"output_type": "stream",
109-
"text": [
110-
"rayclient-hfgputest-1-default.apps.tedbig412.cp.fyre.ibm.com\n"
111-
]
112-
}
113-
],
96+
"outputs": [],
11497
"source": [
115-
"import openshift as oc\n",
11698
"from codeflare_sdk.utils import generate_cert\n",
11799
"\n",
118100
"if local_interactive:\n",
119101
" generate_cert.generate_tls_cert(cluster_name, namespace)\n",
120-
" generate_cert.export_env(cluster_name, namespace)\n",
121-
"\n",
122-
"with oc.project(namespace):\n",
123-
" routes=oc.selector(\"route\").objects()\n",
124-
" rayclient_url=\"\"\n",
125-
" for r in routes:\n",
126-
" if \"rayclient\" in r.name():\n",
127-
" rayclient_url=r.model.spec.host\n",
128-
"print(rayclient_url)"
102+
" generate_cert.export_env(cluster_name, namespace)"
129103
]
130104
},
131105
{
132106
"cell_type": "code",
133-
"execution_count": 12,
107+
"execution_count": 6,
134108
"id": "9483bb98-33b3-4beb-9b15-163d7e76c1d7",
135109
"metadata": {
136110
"scrolled": true,
@@ -141,15 +115,15 @@
141115
"name": "stderr",
142116
"output_type": "stream",
143117
"text": [
144-
"2023-05-31 14:12:37,816\tINFO client_builder.py:251 -- Passing the following kwargs to ray.init() on the server: logging_level\n",
145-
"2023-05-31 14:12:37,820\tDEBUG worker.py:378 -- client gRPC channel state change: ChannelConnectivity.IDLE\n",
146-
"2023-05-31 14:12:38,034\tDEBUG worker.py:378 -- client gRPC channel state change: ChannelConnectivity.CONNECTING\n",
147-
"2023-05-31 14:12:38,246\tDEBUG worker.py:378 -- client gRPC channel state change: ChannelConnectivity.READY\n",
148-
"2023-05-31 14:12:38,290\tDEBUG worker.py:807 -- Pinging server.\n",
149-
"2023-05-31 14:12:40,521\tDEBUG worker.py:640 -- Retaining 00ffffffffffffffffffffffffffffffffffffff0100000001000000\n",
150-
"2023-05-31 14:12:40,523\tDEBUG worker.py:564 -- Scheduling task get_dashboard_url 0 b'\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00'\n",
151-
"2023-05-31 14:12:40,535\tDEBUG worker.py:640 -- Retaining c8ef45ccd0112571ffffffffffffffffffffffff0100000001000000\n",
152-
"2023-05-31 14:12:41,379\tDEBUG worker.py:636 -- Releasing c8ef45ccd0112571ffffffffffffffffffffffff0100000001000000\n"
118+
"2023-06-27 19:14:16,088\tINFO client_builder.py:251 -- Passing the following kwargs to ray.init() on the server: logging_level\n",
119+
"2023-06-27 19:14:16,100\tDEBUG worker.py:378 -- client gRPC channel state change: ChannelConnectivity.IDLE\n",
120+
"2023-06-27 19:14:16,308\tDEBUG worker.py:378 -- client gRPC channel state change: ChannelConnectivity.CONNECTING\n",
121+
"2023-06-27 19:14:16,434\tDEBUG worker.py:378 -- client gRPC channel state change: ChannelConnectivity.READY\n",
122+
"2023-06-27 19:14:16,436\tDEBUG worker.py:807 -- Pinging server.\n",
123+
"2023-06-27 19:14:18,634\tDEBUG worker.py:640 -- Retaining 00ffffffffffffffffffffffffffffffffffffff0100000001000000\n",
124+
"2023-06-27 19:14:18,635\tDEBUG worker.py:564 -- Scheduling task get_dashboard_url 0 b'\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00'\n",
125+
"2023-06-27 19:14:18,645\tDEBUG worker.py:640 -- Retaining c8ef45ccd0112571ffffffffffffffffffffffff0100000001000000\n",
126+
"2023-06-27 19:14:19,454\tDEBUG worker.py:636 -- Releasing c8ef45ccd0112571ffffffffffffffffffffffff0100000001000000\n"
153127
]
154128
},
155129
{
@@ -190,18 +164,18 @@
190164
" </tr>\n",
191165
" <tr>\n",
192166
" <td style=\"text-align: left\"><b>Dashboard:</b></td>\n",
193-
" <td style=\"text-align: left\"><b><a href=\"http://10.254.12.141:8265\" target=\"_blank\">http://10.254.12.141:8265</a></b></td>\n",
167+
" <td style=\"text-align: left\"><b><a href=\"http://10.254.20.41:8265\" target=\"_blank\">http://10.254.20.41:8265</a></b></td>\n",
194168
"</tr>\n",
195169
"\n",
196170
" </table>\n",
197171
" </div>\n",
198172
"</div>\n"
199173
],
200174
"text/plain": [
201-
"ClientContext(dashboard_url='10.254.12.141:8265', python_version='3.8.13', ray_version='2.1.0', ray_commit='23f34d948dae8de9b168667ab27e6cf940b3ae85', protocol_version='2022-10-05', _num_clients=1, _context_to_restore=<ray.util.client._ClientContext object at 0x10e5d2bb0>)"
175+
"ClientContext(dashboard_url='10.254.20.41:8265', python_version='3.8.13', ray_version='2.1.0', ray_commit='23f34d948dae8de9b168667ab27e6cf940b3ae85', protocol_version='2022-10-05', _num_clients=1, _context_to_restore=<ray.util.client._ClientContext object at 0x108ca2730>)"
202176
]
203177
},
204-
"execution_count": 12,
178+
"execution_count": 6,
205179
"metadata": {},
206180
"output_type": "execute_result"
207181
}
@@ -210,12 +184,12 @@
210184
"import ray\n",
211185
"\n",
212186
"ray.shutdown()\n",
213-
"ray.init(address=f\"ray://{rayclient_url}\", logging_level=\"DEBUG\")"
187+
"ray.init(address=cluster.local_client_url(), logging_level=\"DEBUG\")"
214188
]
215189
},
216190
{
217191
"cell_type": "code",
218-
"execution_count": 13,
192+
"execution_count": 7,
219193
"id": "3436eb4a-217c-4109-a3c3-309fda7e2442",
220194
"metadata": {},
221195
"outputs": [],
@@ -239,7 +213,7 @@
239213
},
240214
{
241215
"cell_type": "code",
242-
"execution_count": 14,
216+
"execution_count": 8,
243217
"id": "5cca1874-2be3-4631-ae48-9adfa45e3af3",
244218
"metadata": {
245219
"scrolled": true,
@@ -250,8 +224,8 @@
250224
"name": "stderr",
251225
"output_type": "stream",
252226
"text": [
253-
"2023-05-31 14:13:29,868\tDEBUG worker.py:640 -- Retaining 00ffffffffffffffffffffffffffffffffffffff0100000002000000\n",
254-
"2023-05-31 14:13:29,870\tDEBUG worker.py:564 -- Scheduling task heavy_calculation 0 b'\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x01\\x00\\x00\\x00\\x02\\x00\\x00\\x00'\n"
227+
"2023-06-27 19:14:28,222\tDEBUG worker.py:640 -- Retaining 00ffffffffffffffffffffffffffffffffffffff0100000002000000\n",
228+
"2023-06-27 19:14:28,222\tDEBUG worker.py:564 -- Scheduling task heavy_calculation 0 b'\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x01\\x00\\x00\\x00\\x02\\x00\\x00\\x00'\n"
255229
]
256230
}
257231
],
@@ -261,16 +235,16 @@
261235
},
262236
{
263237
"cell_type": "code",
264-
"execution_count": 15,
238+
"execution_count": 9,
265239
"id": "01172c29-e8bf-41ef-8db5-eccb07906111",
266240
"metadata": {},
267241
"outputs": [
268242
{
269243
"name": "stderr",
270244
"output_type": "stream",
271245
"text": [
272-
"2023-05-31 14:13:32,643\tDEBUG worker.py:640 -- Retaining 16310a0f0a45af5cffffffffffffffffffffffff0100000001000000\n",
273-
"2023-05-31 14:13:34,677\tDEBUG worker.py:439 -- Internal retry for get [ClientObjectRef(16310a0f0a45af5cffffffffffffffffffffffff0100000001000000)]\n"
246+
"2023-06-27 19:14:29,202\tDEBUG worker.py:640 -- Retaining 16310a0f0a45af5cffffffffffffffffffffffff0100000001000000\n",
247+
"2023-06-27 19:14:31,224\tDEBUG worker.py:439 -- Internal retry for get [ClientObjectRef(16310a0f0a45af5cffffffffffffffffffffffff0100000001000000)]\n"
274248
]
275249
},
276250
{
@@ -279,7 +253,7 @@
279253
"1789.4644387076714"
280254
]
281255
},
282-
"execution_count": 15,
256+
"execution_count": 9,
283257
"metadata": {},
284258
"output_type": "execute_result"
285259
}
@@ -290,18 +264,18 @@
290264
},
291265
{
292266
"cell_type": "code",
293-
"execution_count": 16,
267+
"execution_count": 10,
294268
"id": "9e79b547-a457-4232-b77d-19147067b972",
295269
"metadata": {},
296270
"outputs": [
297271
{
298272
"name": "stderr",
299273
"output_type": "stream",
300274
"text": [
301-
"2023-05-31 14:13:37,659\tDEBUG dataclient.py:287 -- Got unawaited response connection_cleanup {\n",
275+
"2023-06-27 19:14:33,161\tDEBUG dataclient.py:287 -- Got unawaited response connection_cleanup {\n",
302276
"}\n",
303277
"\n",
304-
"2023-05-31 14:13:38,681\tDEBUG dataclient.py:278 -- Shutting down data channel.\n"
278+
"2023-06-27 19:14:34,460\tDEBUG dataclient.py:278 -- Shutting down data channel.\n"
305279
]
306280
}
307281
],
@@ -312,7 +286,7 @@
312286
},
313287
{
314288
"cell_type": "code",
315-
"execution_count": 17,
289+
"execution_count": 11,
316290
"id": "2c198f1f-68bf-43ff-a148-02b5cb000ff2",
317291
"metadata": {},
318292
"outputs": [],

Diff for: ‎pyproject.toml

+2-1
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,10 @@ python = "^3.7"
2424
openshift-client = "1.0.18"
2525
rich = "^12.5"
2626
ray = {version = "2.5.0", extras = ["default"]}
27-
kubernetes = "25.3.0"
27+
kubernetes = ">= 25.3.0, < 27"
2828
codeflare-torchx = "0.6.0.dev0"
2929
cryptography = "40.0.2"
30+
executing = "1.2.0"
3031

3132
[tool.poetry.group.docs]
3233
optional = true

Diff for: ‎requirements.txt

+2
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,5 @@ ray[default]==2.5.0
44
kubernetes>=25.3.0,<27
55
codeflare-torchx==0.6.0.dev0
66
pydantic<2 # 2.0+ broke ray[default] see detail: https://github.com/ray-project/ray/pull/37000
7+
cryptography==40.0.2
8+
executing==1.2.0

0 commit comments

Comments
 (0)