From f90c1ce6b53b525bd1e92df64a1b90ce9182aec7 Mon Sep 17 00:00:00 2001 From: hdser Date: Wed, 25 Sep 2024 11:34:52 +0200 Subject: [PATCH] Configs --- .env.test | 5 +++ Dockerfile | 19 +++++++++-- docker-compose.yml | 34 +++++++++++++++++++ .../power_consumption/consensus_power.sql | 28 ++++++--------- profiles.yml | 34 ++++++++++++++++--- requirements.txt | 3 ++ 6 files changed, 97 insertions(+), 26 deletions(-) create mode 100644 .env.test create mode 100644 docker-compose.yml diff --git a/.env.test b/.env.test new file mode 100644 index 0000000..e50fa0d --- /dev/null +++ b/.env.test @@ -0,0 +1,5 @@ +CLICKHOUSE_URL +CLICKHOUSE_PORT +CLICKHOUSE_USER +CLICKHOUSE_PASSWORD +CLICKHOUSE_SECURE \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 8853e51..be64c5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,15 +7,28 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +# Create a non-root user +ARG USER_ID=1000 +ARG GROUP_ID=1000 +RUN addgroup --gid $GROUP_ID appgroup && \ + adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID appuser + # Setup the working directory WORKDIR /app +RUN chown -R appuser:appgroup /app && chmod -R 755 /app + +# Switch to non-root user +USER appuser # Copy the Python requirements and install them -COPY /requirements.txt /app/requirements.txt -RUN pip install -r /app/requirements.txt +COPY requirements.txt /app/requirements.txt +RUN pip install --user -r /app/requirements.txt # Set environment variable to specify the DBT project path ENV DBT_PROJECT_PATH /app/src # Optionally expose a port for dbt docs if needed -EXPOSE 8080 \ No newline at end of file +EXPOSE 8080 + +# Set PATH to include user-level binaries +ENV PATH=/home/appuser/.local/bin:$PATH diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..fc3ca5d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,34 @@ +services: + dbt: + container_name: dbt + build: + context: . + dockerfile: Dockerfile + args: + USER_ID: ${USER_ID:-1000} + GROUP_ID: ${GROUP_ID:-1000} + restart: unless-stopped + ports: + - "8080:8000" + command: > + /bin/bash -c "exec python -m http.server 8000 --directory logs & tail -f /dev/null" + environment: + CLICKHOUSE_USER: ${CLICKHOUSE_USER} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} + CLICKHOUSE_URL: ${CLICKHOUSE_URL} + CLICKHOUSE_PORT: ${CLICKHOUSE_PORT} + CLICKHOUSE_SECURE: ${CLICKHOUSE_SECURE} + volumes: + - type: bind + source: ./ + target: /app + consistency: cached + - ./profiles.yml:/home/appuser/.dbt/profiles.yml + working_dir: /app + user: appuser + networks: + - dbt_net + + +networks: + dbt_net: diff --git a/models/transformations/power_consumption/consensus_power.sql b/models/transformations/power_consumption/consensus_power.sql index 4053e1b..4c4526e 100644 --- a/models/transformations/power_consumption/consensus_power.sql +++ b/models/transformations/power_consumption/consensus_power.sql @@ -1,22 +1,14 @@ WITH consensus_power AS ( - SELECT * FROM ( - VALUES - (4, 'Lighthouse', 2.75), - (5, 'Lighthouse', 3.14), - (6, 'Lighthouse', 18.84), - (4, 'Teku', 3.71), - (5, 'Teku', 3.32), - (6, 'Teku', 27.46), - (4, 'Lodestar', 3.14), - (5, 'Lodestar', 3.89), - (6, 'Lodestar', 33.55), - (4, 'Nimbus', 1.67), - (5, 'Nimbus', 2.08), - (6, 'Nimbus', 17.11), - (4, 'Prysm', 3.51), - (5, 'Prysm', 2.87), - (6, 'Prysm', 24.33) - ) AS t(type, client, mean) + SELECT + type, + client, + mean + FROM ( + SELECT + arrayJoin([4, 5, 6, 4, 5, 6, 4, 5, 6, 4, 5, 6, 4, 5, 6]) AS type, + arrayJoin(['Lighthouse', 'Lighthouse', 'Lighthouse', 'Teku', 'Teku', 'Teku', 'Lodestar', 'Lodestar', 'Lodestar', 'Nimbus', 'Nimbus', 'Nimbus', 'Prysm', 'Prysm', 'Prysm']) AS client, + arrayJoin([2.75, 3.14, 18.84, 3.71, 3.32, 27.46, 3.14, 3.89, 33.55, 1.67, 2.08, 17.11, 3.51, 2.87, 24.33]) AS mean + ) ) SELECT * FROM consensus_power \ No newline at end of file diff --git a/profiles.yml b/profiles.yml index f5aae7e..06ef745 100644 --- a/profiles.yml +++ b/profiles.yml @@ -1,17 +1,41 @@ gnosis_dbt: - target: clickhouse + target: ch-dbt outputs: - clickhouse: # ClickHouse configuration for additional data sourcing + ch-dbt: type: clickhouse - schema: "{{ env_var('CLICKHOUSE_SCHEMA') }}" + schema: "dbt" verify: False host: "{{ env_var('CLICKHOUSE_URL') }}" port: "{{ env_var('CLICKHOUSE_PORT', '8123') | int }}" secure: "{{ (env_var('CLICKHOUSE_SECURE', 'False') | lower) == 'true' }}" - cluster: '{cluster}' - cluster_mode: True user: "{{ env_var('CLICKHOUSE_USER') }}" password: "{{ env_var('CLICKHOUSE_PASSWORD') }}" threads: 40 connect_timeout: 60 # Increase the connect timeout (in seconds) read_timeout: 3000 # Increase the read timeout (in seconds) + + ch-valtrack: + type: clickhouse + schema: "valtrack_preview" + verify: False + host: "{{ env_var('CLICKHOUSE_URL') }}" + port: "{{ env_var('CLICKHOUSE_PORT', '8123') | int }}" + secure: "{{ (env_var('CLICKHOUSE_SECURE', 'False') | lower) == 'true' }}" + user: "{{ env_var('CLICKHOUSE_USER') }}" + password: "{{ env_var('CLICKHOUSE_PASSWORD') }}" + threads: 40 + connect_timeout: 60 + read_timeout: 3000 + + ch-goteth: + type: clickhouse + schema: "goteth_preview" + verify: False + host: "{{ env_var('CLICKHOUSE_URL') }}" + port: "{{ env_var('CLICKHOUSE_PORT', '8123') | int }}" + secure: "{{ (env_var('CLICKHOUSE_SECURE', 'False') | lower) == 'true' }}" + user: "{{ env_var('CLICKHOUSE_USER') }}" + password: "{{ env_var('CLICKHOUSE_PASSWORD') }}" + threads: 40 + connect_timeout: 60 + read_timeout: 3000 diff --git a/requirements.txt b/requirements.txt index e69de29..041490a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,3 @@ +dbt-core==1.8.7 +dbt-clickhouse==1.8.4 +pytest==8.2.1 \ No newline at end of file