Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[GSoC] Add unit tests for tune API #2423

Merged
merged 27 commits into from
Jan 24, 2025
Merged

[GSoC] Add unit tests for tune API #2423

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
5dc6fc5
add unit tests for tune api
helenxie-bit Sep 5, 2024
04a7e39
update
helenxie-bit Sep 5, 2024
8c4d65a
fix format
helenxie-bit Sep 5, 2024
b0195a6
update unit tests and fix api errors
helenxie-bit Sep 5, 2024
a92de67
fix format
helenxie-bit Sep 5, 2024
7b7e347
test
helenxie-bit Sep 5, 2024
e4f7922
test
helenxie-bit Sep 5, 2024
e621fc6
update unit tests
helenxie-bit Sep 9, 2024
9c0a9e6
undo changes to Makefile
helenxie-bit Sep 9, 2024
f5c4bce
delete debug code
helenxie-bit Sep 9, 2024
5ddcc30
fix format
helenxie-bit Sep 9, 2024
4909456
update unit test
helenxie-bit Sep 11, 2024
1e78840
fix format
helenxie-bit Sep 11, 2024
e68fe38
update the version of training operator
helenxie-bit Sep 12, 2024
d3a3404
adjust 'list_namespaced_persistent_volume_claim' to be called with ke…
helenxie-bit Oct 9, 2024
6d5c20e
create constant for namespace when check pvc creation error
helenxie-bit Oct 9, 2024
b25f7ba
add type check for 'trainer_parameters'
helenxie-bit Oct 9, 2024
3ebbe76
fix format
helenxie-bit Oct 9, 2024
0498237
update test names
helenxie-bit Oct 10, 2024
15f6a7a
fix format
helenxie-bit Oct 10, 2024
86db6d5
add verification for key Experiment information & add 'kubeflow-train…
helenxie-bit Oct 22, 2024
b24b44f
rerun tests
helenxie-bit Oct 22, 2024
5dfd1a3
add verification for objective metric name
helenxie-bit Jan 23, 2025
a0dbeeb
resolve conflict
helenxie-bit Jan 23, 2025
018ec33
delete unnecessary changes
helenxie-bit Jan 23, 2025
8cd3d0c
unify objective function
helenxie-bit Jan 23, 2025
b1d07ce
unify objective function
helenxie-bit Jan 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion sdk/python/v1beta1/kubeflow/katib/api/katib_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -570,7 +570,7 @@ class name in this argument.
)
except Exception as e:
pvc_list = self.core_api.list_namespaced_persistent_volume_claim(
namespace
namespace=namespace
)
# Check if the PVC with the specified name exists.
for pvc in pvc_list.items:
Expand Down
336 changes: 336 additions & 0 deletions sdk/python/v1beta1/kubeflow/katib/api/katib_client_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@
from typing import List, Optional
from unittest.mock import Mock, patch

import kubeflow.katib as katib
import kubeflow.katib.katib_api_pb2 as katib_api_pb2
import pytest
import transformers
from kubeflow.katib import (
KatibClient,
V1beta1AlgorithmSpec,
Expand All @@ -16,8 +18,15 @@
V1beta1TrialTemplate,
)
from kubeflow.katib.constants import constants
from kubeflow.storage_initializer.hugging_face import (
HuggingFaceDatasetParams,
HuggingFaceModelParams,
HuggingFaceTrainerParams,
)
from kubernetes.client import V1ObjectMeta

PVC_FAILED = "pvc creation failed"

TEST_RESULT_SUCCESS = "success"


Expand Down Expand Up @@ -57,6 +66,27 @@ def get_observation_log_response(*args, **kwargs):
)


def create_namespaced_persistent_volume_claim_response(*args, **kwargs):
if kwargs.get("namespace") == PVC_FAILED:
raise Exception("PVC creation failed")
else:
return {"metadata": {"name": "tune_test"}}


def list_namespaced_persistent_volume_claim_response(*args, **kwargs):
if kwargs.get("namespace") == PVC_FAILED:
mock_pvc = Mock()
mock_pvc.metadata.name = "pvc_failed"
mock_list = Mock()
mock_list.items = [mock_pvc]
else:
mock_pvc = Mock()
mock_pvc.metadata.name = "tune_test"
mock_list = Mock()
mock_list.items = [mock_pvc]
return mock_list


def generate_trial_template() -> V1beta1TrialTemplate:
trial_spec = {
"apiVersion": "batch/v1",
Expand Down Expand Up @@ -270,6 +300,215 @@ def create_experiment(
]


test_tune_data = [
(
"missing name",
{
"name": None,
"objective": lambda x: print(f"a={x}"),
"parameters": {"a": katib.search.int(min=10, max=100)},
},
ValueError,
),
(
"invalid hybrid parameters - objective and model_provider_parameters",
{
"name": "tune_test",
"objective": lambda x: print(f"a={x}"),
"model_provider_parameters": HuggingFaceModelParams(
model_uri="hf://google-bert/bert-base-cased",
transformer_type=transformers.AutoModelForSequenceClassification,
num_labels=5,
),
},
ValueError,
),
(
"missing parameters - no custom objective or external model tuning",
{
"name": "tune_test",
},
ValueError,
),
(
"missing parameters in custom objective tuning - lack parameters",
{
"name": "tune_test",
"objective": lambda x: print(f"a={x}"),
},
ValueError,
),
(
"missing parameters in custom objective tuning - lack objective",
{
"name": "tune_test",
"parameters": {"a": katib.search.int(min=10, max=100)},
},
ValueError,
),
(
"missing parameters in external model tuning - lack dataset_provider_parameters "
"and trainer_parameters",
{
"name": "tune_test",
"model_provider_parameters": HuggingFaceModelParams(
model_uri="hf://google-bert/bert-base-cased",
transformer_type=transformers.AutoModelForSequenceClassification,
num_labels=5,
),
},
ValueError,
),
(
"missing parameters in external model tuning - lack model_provider_parameters "
"and trainer_parameters",
{
"name": "tune_test",
"dataset_provider_parameters": HuggingFaceDatasetParams(
repo_id="yelp_review_full",
split="train[:3000]",
),
},
ValueError,
),
(
"missing parameters in external model tuning - lack model_provider_parameters "
"and dataset_provider_parameters",
{
"name": "tune_test",
"trainer_parameters": HuggingFaceTrainerParams(
training_parameters=transformers.TrainingArguments(
output_dir="test_tune_api",
learning_rate=katib.search.double(min=1e-05, max=5e-05),
),
),
},
ValueError,
),
(
"invalid env_per_trial",
{
"name": "tune_test",
"objective": lambda x: print(f"a={x}"),
"parameters": {"a": katib.search.int(min=10, max=100)},
"env_per_trial": "invalid",
},
ValueError,
),
(
"invalid model_provider_parameters",
{
"name": "tune_test",
"model_provider_parameters": "invalid",
"dataset_provider_parameters": HuggingFaceDatasetParams(
repo_id="yelp_review_full",
split="train[:3000]",
),
"trainer_parameters": HuggingFaceTrainerParams(
training_parameters=transformers.TrainingArguments(
output_dir="test_tune_api",
learning_rate=katib.search.double(min=1e-05, max=5e-05),
),
),
},
ValueError,
),
(
"invalid dataset_provider_parameters",
{
"name": "tune_test",
"model_provider_parameters": HuggingFaceModelParams(
model_uri="hf://google-bert/bert-base-cased",
transformer_type=transformers.AutoModelForSequenceClassification,
num_labels=5,
),
"dataset_provider_parameters": "invalid",
"trainer_parameters": HuggingFaceTrainerParams(
training_parameters=transformers.TrainingArguments(
output_dir="test_tune_api",
learning_rate=katib.search.double(min=1e-05, max=5e-05),
),
),
},
ValueError,
),
(
"invalid trainer_parameters",
{
"name": "tune_test",
"model_provider_parameters": HuggingFaceModelParams(
model_uri="hf://google-bert/bert-base-cased",
transformer_type=transformers.AutoModelForSequenceClassification,
num_labels=5,
),
"dataset_provider_parameters": HuggingFaceDatasetParams(
repo_id="yelp_review_full",
split="train[:3000]",
),
"trainer_parameters": "invalid",
},
ValueError,
),
(
"pvc creation failed",
{
"name": "tune_test",
"namespace": PVC_FAILED,
"model_provider_parameters": HuggingFaceModelParams(
model_uri="hf://google-bert/bert-base-cased",
transformer_type=transformers.AutoModelForSequenceClassification,
num_labels=5,
),
"dataset_provider_parameters": HuggingFaceDatasetParams(
repo_id="yelp_review_full",
split="train[:3000]",
),
"trainer_parameters": HuggingFaceTrainerParams(
training_parameters=transformers.TrainingArguments(
output_dir="test_tune_api",
learning_rate=katib.search.double(min=1e-05, max=5e-05),
),
),
},
RuntimeError,
),
(
"valid flow with custom objective tuning",
{
"name": "tune_test",
"objective": lambda x: print(f"a={x}"),
"parameters": {"a": katib.search.int(min=10, max=100)},
"objective_metric_name": "a",
},
TEST_RESULT_SUCCESS,
),
(
"valid flow with external model tuning",
{
"name": "tune_test",
"model_provider_parameters": HuggingFaceModelParams(
model_uri="hf://google-bert/bert-base-cased",
transformer_type=transformers.AutoModelForSequenceClassification,
num_labels=5,
),
"dataset_provider_parameters": HuggingFaceDatasetParams(
repo_id="yelp_review_full",
split="train[:3000]",
),
"trainer_parameters": HuggingFaceTrainerParams(
training_parameters=transformers.TrainingArguments(
output_dir="test_tune_api",
learning_rate=katib.search.double(min=1e-05, max=5e-05),
),
),
"objective_metric_name": "train_loss",
"objective_type": "minimize",
},
TEST_RESULT_SUCCESS,
),
]


@pytest.fixture
def katib_client():
with patch(
Expand All @@ -284,6 +523,16 @@ def katib_client():
return_value=Mock(
GetObservationLog=Mock(side_effect=get_observation_log_response)
),
), patch(
"kubernetes.client.CoreV1Api",
return_value=Mock(
create_namespaced_persistent_volume_claim=Mock(
side_effect=create_namespaced_persistent_volume_claim_response
),
list_namespaced_persistent_volume_claim=Mock(
side_effect=list_namespaced_persistent_volume_claim_response
),
),
):
client = KatibClient()
yield client
Expand Down Expand Up @@ -320,3 +569,90 @@ def test_get_trial_metrics(katib_client, test_name, kwargs, expected_output):
except Exception as e:
assert type(e) is expected_output
print("test execution complete")


@pytest.mark.parametrize("test_name,kwargs,expected_output", test_tune_data)
def test_tune(katib_client, test_name, kwargs, expected_output):
"""
test tune function of katib client
"""
print("\n\nExecuting test:", test_name)

with patch.object(
katib_client, "create_experiment", return_value=Mock()
) as mock_create_experiment:
try:
katib_client.tune(**kwargs)
mock_create_experiment.assert_called_once()

if expected_output == TEST_RESULT_SUCCESS:
assert expected_output == TEST_RESULT_SUCCESS
call_args = mock_create_experiment.call_args
experiment = call_args[0][0]

if test_name == "valid flow with custom objective tuning":
# Verify input_params
args_content = "".join(
experiment.spec.trial_template.trial_spec.spec.template.spec.containers[
0
].args
)
assert "'a': '${trialParameters.a}'" in args_content
# Verify trial_params
assert experiment.spec.trial_template.trial_parameters == [
V1beta1TrialParameterSpec(name="a", reference="a"),
]
# Verify experiment_params
assert experiment.spec.parameters == [
V1beta1ParameterSpec(
name="a",
parameter_type="int",
feasible_space=V1beta1FeasibleSpace(min="10", max="100"),
),
]
Comment on lines +593 to +612
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be better if we also verify the objective_metric_name parameter.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for your suggestions! I will add that.

# Verify objective_spec
assert experiment.spec.objective == V1beta1ObjectiveSpec(
type="maximize",
objective_metric_name="a",
additional_metric_names=[],
)

elif test_name == "valid flow with external model tuning":
# Verify input_params
args_content = "".join(
experiment.spec.trial_template.trial_spec.spec.pytorch_replica_specs[
"Master"
]
.template.spec.containers[0]
.args
)
assert (
'"learning_rate": "${trialParameters.learning_rate}"'
in args_content
)
# Verify trial_params
assert experiment.spec.trial_template.trial_parameters == [
V1beta1TrialParameterSpec(
name="learning_rate", reference="learning_rate"
),
]
# Verify experiment_params
assert experiment.spec.parameters == [
V1beta1ParameterSpec(
name="learning_rate",
parameter_type="double",
feasible_space=V1beta1FeasibleSpace(
min="1e-05", max="5e-05"
),
),
]
# Verify objective_spec
assert experiment.spec.objective == V1beta1ObjectiveSpec(
type="minimize",
objective_metric_name="train_loss",
additional_metric_names=[],
)

except Exception as e:
assert type(e) is expected_output
print("test execution complete")
1 change: 1 addition & 0 deletions test/unit/v1beta1/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
grpcio-testing==1.64.1
pytest==7.2.0
kubeflow-training[huggingface]==1.8.1
Loading