From 95cbf07d246b9f2fb5af292c1fb65f523a9c6d83 Mon Sep 17 00:00:00 2001 From: pmahindrakar-oss Date: Tue, 6 Jun 2023 13:43:03 -0700 Subject: [PATCH] Bigquery docker image (#995) * Added bigquery dockerfile Signed-off-by: pmahindrakar-oss * removed opencv dependencies Signed-off-by: pmahindrakar-oss --------- Signed-off-by: pmahindrakar-oss --- cookbook/integrations/gcp/bigquery/Dockerfile | 37 +++++++++++++++++++ cookbook/integrations/gcp/bigquery/Makefile | 14 ------- .../integrations/gcp/bigquery/bigquery.py | 6 +-- 3 files changed, 40 insertions(+), 17 deletions(-) create mode 100644 cookbook/integrations/gcp/bigquery/Dockerfile diff --git a/cookbook/integrations/gcp/bigquery/Dockerfile b/cookbook/integrations/gcp/bigquery/Dockerfile new file mode 100644 index 000000000..184f8b585 --- /dev/null +++ b/cookbook/integrations/gcp/bigquery/Dockerfile @@ -0,0 +1,37 @@ +FROM python:3.8-slim-buster +LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks + +WORKDIR /root +ENV VENV /opt/venv +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV PYTHONPATH /root + +RUN apt-get update && apt-get install -y build-essential curl + +WORKDIR /opt +RUN curl https://sdk.cloud.google.com > install.sh +RUN bash /opt/install.sh --install-dir=/opt +ENV PATH $PATH:/opt/google-cloud-sdk/bin +WORKDIR /root + +ENV VENV /opt/venv +# Virtual environment +RUN python3 -m venv ${VENV} +ENV PATH="${VENV}/bin:$PATH" + +# Install Python dependencies +COPY bigquery/requirements.txt /root +RUN pip install -r /root/requirements.txt + +# Copy the makefile targets to expose on the container. This makes it easier to register +COPY in_container.mk /root/Makefile +COPY bigquery/sandbox.config /root + +# Copy the actual code +COPY bigquery/ /root/bigquery + +# This tag is supplied by the build script and will be used to determine the version +# when registering tasks, workflows, and launch plans +ARG tag +ENV FLYTE_INTERNAL_IMAGE $tag diff --git a/cookbook/integrations/gcp/bigquery/Makefile b/cookbook/integrations/gcp/bigquery/Makefile index 1ad0ab705..e2902c492 100644 --- a/cookbook/integrations/gcp/bigquery/Makefile +++ b/cookbook/integrations/gcp/bigquery/Makefile @@ -2,17 +2,3 @@ PREFIX=bigquery include ../../../common/common.mk include ../../../common/leaf.mk -.PHONY: docker_build -docker_build: ; - -.PHONY: docker_push -docker_push: ; - -.PHONY: register -register: ; - -.PHONY: serialize -serialize: ; - -.PHONY: docker_push -docker_push: ; diff --git a/cookbook/integrations/gcp/bigquery/bigquery.py b/cookbook/integrations/gcp/bigquery/bigquery.py index 4552b9f17..9ce56db07 100644 --- a/cookbook/integrations/gcp/bigquery/bigquery.py +++ b/cookbook/integrations/gcp/bigquery/bigquery.py @@ -20,7 +20,7 @@ name="sql.bigquery.no_io", inputs={}, query_template="SELECT 1", - task_config=BigQueryConfig(ProjectID="flyte", Location="us-west1-b"), + task_config=BigQueryConfig(ProjectID="flyte"), ) @@ -46,8 +46,8 @@ def no_io_wf(): # Define inputs as well as their types that can be used to customize the query. inputs=kwtypes(version=int), output_structured_dataset_type=DogeCoinDataset, - task_config=BigQueryConfig(ProjectID="flyte", Location="us-west1-b"), - query_template="SELECT * FROM `bigquery-public-data.crypto_dogecoin.transactions` WHERE @version = 1 LIMIT 10;", + task_config=BigQueryConfig(ProjectID="flyte"), + query_template="SELECT * FROM `bigquery-public-data.crypto_dogecoin.transactions` WHERE version = 1 LIMIT 10;", )