From 8c896aeffc3c799b3295e4fa5619edb7e72a39f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20P=C3=A9rez?= Date: Tue, 10 Sep 2024 12:44:58 -0400 Subject: [PATCH] [spark] Stop checking if volume to mount in k8s exists locally (#148) It's entirely possible (and not an error) to create a podspec that contains paths to volume-mount that do not exist wherever service_configuration_lib is being invoked from. As long as we ensure that we don't double-add things (which is only a problem due to the way that the env var keys are constructed here), skipping the existence check should be fine. --- service_configuration_lib/spark_config.py | 5 ++--- tests/spark_config_test.py | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/service_configuration_lib/spark_config.py b/service_configuration_lib/spark_config.py index 5c11293..108f620 100644 --- a/service_configuration_lib/spark_config.py +++ b/service_configuration_lib/spark_config.py @@ -223,13 +223,12 @@ def _get_k8s_docker_volumes_conf( for volume in k8s_volumes: host_path, container_path, mode = volume['hostPath'], volume['containerPath'], volume['mode'] - if os.path.exists(host_path) and host_path not in mounted_volumes: + if host_path not in mounted_volumes: env.update(_get_k8s_volume(host_path, container_path, mode)) mounted_volumes.add(host_path) else: log.warning( - f'Path {host_path} does not exist on this host or it has already been mounted.' - ' Skipping this bindings.', + f'Path {host_path} has already been mounted. Skipping this binding.', ) return env diff --git a/tests/spark_config_test.py b/tests/spark_config_test.py index 60150ed..6bb456e 100644 --- a/tests/spark_config_test.py +++ b/tests/spark_config_test.py @@ -1216,7 +1216,6 @@ def _get_k8s_base_volumes(self): """Helper needed to allow tests to pass in github CI checks.""" return [ volume for volume in spark_config.K8S_BASE_VOLUMES - if os.path.exists(volume['containerPath']) ] @pytest.fixture