From 1bdbba01b51b82f8d97ae05d00b21813505833b4 Mon Sep 17 00:00:00 2001 From: Andrey Velichkevich Date: Tue, 7 Jan 2025 01:39:53 +0000 Subject: [PATCH] V1: Fix HF dataset initializer Signed-off-by: Andrey Velichkevich --- .github/workflows/e2e-test-train-api.yaml | 2 +- sdk/python/kubeflow/storage_initializer/requirements.txt | 3 +-- sdk/python/kubeflow/trainer/Dockerfile.cpu | 1 - sdk/python/kubeflow/trainer/requirements.txt | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e-test-train-api.yaml b/.github/workflows/e2e-test-train-api.yaml index 434f5cca80..045c3b19e2 100644 --- a/.github/workflows/e2e-test-train-api.yaml +++ b/.github/workflows/e2e-test-train-api.yaml @@ -58,4 +58,4 @@ jobs: pytest -s sdk/python/test/e2e-fine-tune-llm/test_e2e_pytorch_fine_tune_llm.py --log-cli-level=debug env: STORAGE_INITIALIZER_IMAGE: kubeflowtraining/storage-initializer:test - TRAINER_TRANSFORMER_IMAGE_DEFAULT: kubeflowtraining/trainer:test + TRAINER_TRANSFORMER_IMAGE: kubeflowtraining/trainer:test diff --git a/sdk/python/kubeflow/storage_initializer/requirements.txt b/sdk/python/kubeflow/storage_initializer/requirements.txt index 24ad9f7675..4aa157c00c 100644 --- a/sdk/python/kubeflow/storage_initializer/requirements.txt +++ b/sdk/python/kubeflow/storage_initializer/requirements.txt @@ -1,5 +1,4 @@ peft==0.3.0 -datasets==2.15.0 +datasets==2.21.0 transformers==4.38.0 boto3==1.33.9 -huggingface_hub==0.23.4 diff --git a/sdk/python/kubeflow/trainer/Dockerfile.cpu b/sdk/python/kubeflow/trainer/Dockerfile.cpu index 3cc5009b89..20ae09ceb2 100644 --- a/sdk/python/kubeflow/trainer/Dockerfile.cpu +++ b/sdk/python/kubeflow/trainer/Dockerfile.cpu @@ -8,7 +8,6 @@ WORKDIR /app COPY requirements.txt /app/requirements.txt # Install any needed packages specified in requirements.txt -RUN pip install --no-cache-dir torch==2.5.1 RUN pip install --no-cache-dir -r requirements.txt # Copy the Python package and its source code into the container diff --git a/sdk/python/kubeflow/trainer/requirements.txt b/sdk/python/kubeflow/trainer/requirements.txt index 59171ab910..f820ccddc8 100644 --- a/sdk/python/kubeflow/trainer/requirements.txt +++ b/sdk/python/kubeflow/trainer/requirements.txt @@ -1,4 +1,4 @@ peft==0.3.0 -datasets==2.15.0 +datasets==2.21.0 transformers==4.38.0 accelerate==0.28.0