test(sdk): add pod error msg output.

Signed-off-by: Electronic-Waste <[email protected]>
kubeflow · Jul 31, 2024 · 5b67142 · 5b67142
1 parent d8b0120
commit 5b67142
Showing 1 changed file with 8 additions and 4 deletions.
diff --git a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py
@@ -827,10 +827,14 @@ def wait_for_experiment_condition(
             logger.debug(
                 f"Pods in kubeflow namespace:\n{result.stdout}"
             )
-            result = subprocess.run(['kubectl', 'get', 'pod', '-n', 'default'])
-            logger.debug(
-                f"Pods in default namespace:\n{result.stdout}"
-            )
+            for line in result.stdout.splitlines():
+                if "Error" in line:
+                    pod_name = line.split()[0]
+                    errmsg = subprocess.run(['kubectl', 'describe', 'pod', pod_name, '-n', 'kubeflow'])
+                    logger.debug(
+                        f"Pod {pod_name} in error state:\n{errmsg.stdout}"
+                    )
+
             logger.debug(
                 f"Waiting for Experiment: {namespace}/{name} to reach {expected_condition} condition\n\n\n"
             )