use custom image

Author: kryanbeane

.github/workflows/e2e_tests.yaml

@@ -17,7 +17,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CODEFLARE_OPERATOR_IMG: "quay.io/project-codeflare/codeflare-operator:dev"
+  CODEFLARE_OPERATOR_IMG: "quay.io/kryanbeane/codeflare-operator:dev"
 
 jobs:
   kubernetes:
@@ -108,6 +108,12 @@ jobs:
           kubectl create clusterrolebinding sdk-user-pod-creator --clusterrole=pod-creator --user=sdk-user
           kubectl config use-context sdk-user
 
+      - name: Setup tmate session for debugging
+        if: always()
+        uses: mxschmitt/action-tmate@v3
+        with:
+          detached: true
+          
       - name: Run e2e tests
         run: |
           export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}

src/codeflare_sdk/common/utils/generate_cert.py

@@ -230,22 +230,41 @@ def generate_tls_cert(cluster_name, namespace, days=30):
 
 def export_env(cluster_name, namespace):
     """
-    Sets environment variables to configure TLS for a Ray cluster.
+    Sets environment variables to configure TLS for a Ray client connection when mTLS is enabled.
+
+    The `tls.crt` and `tls.key` files generated by `generate_tls_cert` are client-side credentials,
+    signed by the cluster's CA. `ca.crt` is the cluster's CA certificate.
+
+    This function sets:
+    - `RAY_USE_TLS="1"` to enable TLS.
+    - `RAY_TLS_CA_CERT` to the path of `ca.crt` for server certificate verification.
+    - `RAY_TLS_CLIENT_CERT` and `RAY_TLS_CLIENT_KEY` to the paths of the client's `tls.crt`
+      and `tls.key` respectively, for client authentication by the server.
+    - `RAY_TLS_SERVER_CERT` and `RAY_TLS_SERVER_KEY` are also set to the client's `tls.crt`
+      and `tls.key`. This is maintained based on previous observations that these might be
+      utilized by certain Ray client setups, ensuring broad compatibility.
 
     Args:
         cluster_name (str):
             The name of the Ray cluster.
         namespace (str):
             The Kubernetes namespace where the Ray cluster is located.
-
-    Environment Variables Set:
-        - RAY_USE_TLS: Enables TLS for Ray.
-        - RAY_TLS_SERVER_CERT: Path to the TLS server certificate.
-        - RAY_TLS_SERVER_KEY: Path to the TLS server private key.
-        - RAY_TLS_CA_CERT: Path to the CA certificate.
     """
     tls_dir = os.path.join(os.getcwd(), f"tls-{cluster_name}-{namespace}")
+    client_cert_path = os.path.join(tls_dir, "tls.crt")
+    client_key_path = os.path.join(tls_dir, "tls.key")
+    ca_cert_path = os.path.join(tls_dir, "ca.crt")
+
     os.environ["RAY_USE_TLS"] = "1"
-    os.environ["RAY_TLS_SERVER_CERT"] = os.path.join(tls_dir, "tls.crt")
-    os.environ["RAY_TLS_SERVER_KEY"] = os.path.join(tls_dir, "tls.key")
-    os.environ["RAY_TLS_CA_CERT"] = os.path.join(tls_dir, "ca.crt")
+
+    # CA certificate for verifying the server
+    os.environ["RAY_TLS_CA_CERT"] = ca_cert_path
+
+    # Standard mTLS client variables: client's own certificate and key
+    os.environ["RAY_TLS_CLIENT_CERT"] = client_cert_path
+    os.environ["RAY_TLS_CLIENT_KEY"] = client_key_path
+
+    # Also set RAY_TLS_SERVER_CERT/KEY to client cert/key, maintaining previous setup style
+    # while ensuring client certs are explicitly available via RAY_TLS_CLIENT_*
+    os.environ["RAY_TLS_SERVER_CERT"] = client_cert_path
+    os.environ["RAY_TLS_SERVER_KEY"] = client_key_path

tests/e2e/local_interactive_sdk_kind_test.py

@@ -8,7 +8,7 @@
 import pytest
 import ray
 import math
-
+import time
 from support import *
 
 
@@ -66,6 +66,25 @@ def run_local_interactives(
         print(cluster.local_client_url())
 
         ray.shutdown()
+
+        print("RAY DEBUGGING")
+        print("\n========== PYTHON DEBUG INFO ==========")
+        print(f"Ray cluster client URL: {cluster.local_client_url()}")
+        print(f"Cluster name: {cluster_name}")
+        print(f"Current working directory: {os.getcwd()}")
+        print(f"Cluster: {cluster}")
+        print(f"Cluster namespace: {self.namespace}")
+        print(f"Cluster name: {cluster_name}")
+        print(f"Cluster config: {cluster.config}")
+        print(f"Cluster config namespace: {cluster.config.namespace}")
+        print(f"Cluster config name: {cluster.config.name}")
+        print(f"Cluster config num_workers: {cluster.config.num_workers}")
+        print(f"Cluster config num_workers: {cluster.config.num_workers}")
+        print("END OF RAY DEBUGGING")
+
+        print("Sleeping for 15 minutes before ray.init for debugging...")
+        time.sleep(900)
+
         ray.init(address=cluster.local_client_url(), logging_level="DEBUG")
 
         @ray.remote(num_gpus=number_of_gpus / 2)