From caaa4559c3eac58173126eddd9f37b9a8642ad26 Mon Sep 17 00:00:00 2001 From: Marek Date: Fri, 14 Feb 2025 12:35:49 +0100 Subject: [PATCH 01/20] Rm `.env` files --- docker/.env | 33 --------------------------- docker/test.env | 60 ------------------------------------------------- 2 files changed, 93 deletions(-) delete mode 100644 docker/.env delete mode 100644 docker/test.env diff --git a/docker/.env b/docker/.env deleted file mode 100644 index 2d96240f23e..00000000000 --- a/docker/.env +++ /dev/null @@ -1,33 +0,0 @@ -RUST_LOG=info -# This variable forces the use of color in the logs -ZEBRA_FORCE_USE_COLOR=1 -LOG_COLOR=true - -### -# Configuration Variables -# These variables are used to configure the zebra node -# Check the entrypoint.sh script for more details -### - -# The config file full path used in the Dockerfile. -ZEBRA_CONF_PATH=/etc/zebrad/zebrad.toml -# [network] -NETWORK=Mainnet -ZEBRA_LISTEN_ADDR=0.0.0.0 -# [consensus] -ZEBRA_CHECKPOINT_SYNC=true -# [state] -# Set this to change the default cached state directory -ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache -# [metrics] -METRICS_ENDPOINT_ADDR=0.0.0.0 -METRICS_ENDPOINT_PORT=9999 -# [tracing] -TRACING_ENDPOINT_ADDR=0.0.0.0 -TRACING_ENDPOINT_PORT=3000 -# [rpc] -RPC_LISTEN_ADDR=0.0.0.0 -# if ${RPC_PORT} is not set, it will use the default value for the current network -RPC_PORT=8232 - diff --git a/docker/test.env b/docker/test.env deleted file mode 100644 index fd2a7c876b7..00000000000 --- a/docker/test.env +++ /dev/null @@ -1,60 +0,0 @@ -### -# Configuration Variables -# These variables are used to configure the zebra node -# Check the entrypoint.sh script for more details -### - -# Set this to change the default log level (must be set at build time) -RUST_LOG=info -# This variable forces the use of color in the logs -ZEBRA_FORCE_USE_COLOR=1 -LOG_COLOR=true -# Path to the config file. This variable has a default set in entrypoint.sh -# ZEBRA_CONF_PATH=/etc/zebrad/zebrad.toml -# [network] -NETWORK=Mainnet -# [state] -# Set this to change the default cached state directory -ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache -# [tracing] -LOG_COLOR=false -TRACING_ENDPOINT_ADDR=0.0.0.0 -TRACING_ENDPOINT_PORT=3000 - -#### -# Test Variables -# These variables are used to run tests in the Dockerfile -# Check the entrypoint.sh script for more details -#### - -# Unit tests -# TODO: These variables are evaluated to any value, even setting a NULL value will evaluate to true -# TEST_FAKE_ACTIVATION_HEIGHTS= -# ZEBRA_SKIP_NETWORK_TESTS -# ZEBRA_SKIP_IPV6_TESTS -RUN_ALL_TESTS= -RUN_ALL_EXPERIMENTAL_TESTS= -TEST_ZEBRA_EMPTY_SYNC= -ZEBRA_TEST_LIGHTWALLETD= -# Integration Tests -# Most of these tests require a cached state directory to save the network state -TEST_DISK_REBUILD= -# These tests needs a Zebra cached state -TEST_CHECKPOINT_SYNC= -GENERATE_CHECKPOINTS_MAINNET= -GENERATE_CHECKPOINTS_TESTNET= -TEST_UPDATE_SYNC= -# These tests need a Lightwalletd binary + a Zebra cached state -TEST_LWD_RPC_CALL= -TEST_GET_BLOCK_TEMPLATE= -TEST_SUBMIT_BLOCK= -# These tests need a Lightwalletd binary + Lightwalletd cached state + a Zebra cached state -TEST_LWD_UPDATE_SYNC= -TEST_LWD_GRPC= -TEST_LWD_TRANSACTIONS= -# Full sync tests -# These tests could take a long time to run, depending on the network -FULL_SYNC_MAINNET_TIMEOUT_MINUTES= -FULL_SYNC_TESTNET_TIMEOUT_MINUTES= -TEST_LWD_FULL_SYNC= From 427cb588c9936afb61709867ac68b201fe70fbdb Mon Sep 17 00:00:00 2001 From: Marek Date: Fri, 14 Feb 2025 12:36:15 +0100 Subject: [PATCH 02/20] Update `mining-docker.md` --- book/src/user/mining-docker.md | 47 ++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/book/src/user/mining-docker.md b/book/src/user/mining-docker.md index 96f47918763..a65a32002ea 100644 --- a/book/src/user/mining-docker.md +++ b/book/src/user/mining-docker.md @@ -1,40 +1,43 @@ # Mining with Zebra in Docker -Zebra's [Docker images](https://hub.docker.com/r/zfnd/zebra/tags) can be used for your mining -operations. If you don't have Docker, see the -[manual configuration instructions](https://zebra.zfnd.org/user/mining.html). +Zebra's [Docker images](https://hub.docker.com/r/zfnd/zebra/tags) can be used +for your mining operations. If you don't have Docker, see the [manual +configuration instructions](https://zebra.zfnd.org/user/mining.html). Using docker, you can start mining by running: ```bash -docker run -e MINER_ADDRESS="t3dvVE3SQEi7kqNzwrfNePxZ1d4hUyztBA1" -p 8232:8232 zfnd/zebra:latest +docker run --name -zebra_local -e MINER_ADDRESS="t3dvVE3SQEi7kqNzwrfNePxZ1d4hUyztBA1" -e ZEBRA_RPC_PORT=8232 -p 8232:8232 zfnd/zebra:latest ``` -This command starts a container on Mainnet and binds port 8232 on your Docker host. If you -want to start generating blocks, you need to let Zebra sync first. +This command starts a container on Mainnet and binds port 8232 on your Docker +host. If you want to start generating blocks, you need to let Zebra sync first. Note that you must pass the address for your mining rewards via the `MINER_ADDRESS` environment variable when you are starting the container, as we -did with the ZF funding stream address above. The address we used starts with the prefix `t1`, -meaning it is a Mainnet P2PKH address. Please remember to set your own address -for the rewards. +did with the ZF funding stream address above. The address we used starts with +the prefix `t1`, meaning it is a Mainnet P2PKH address. Please remember to set +your own address for the rewards. The port we mapped between the container and the host with the `-p` flag in the example above is Zebra's default Mainnet RPC port. Instead of listing the environment variables on the command line, you can use -Docker's `--env-file` flag to specify a file containing the variables. You -can find more info here +Docker's `--env-file` flag to specify a file containing the variables. You can +find more info here https://docs.docker.com/engine/reference/commandline/run/#env. -## Mining on Testnet +If you don't want to set any environment variables, you can edit the +`docker/default_zebra_config.toml` file, and pass it to Zebra before starting +the container. There's an example in `docker/docker-compose.yml` of how to do +that. If you want to mine on Testnet, you need to set the `NETWORK` environment variable to `Testnet` and use a Testnet address for the rewards. For example, running ```bash -docker run -e NETWORK="Testnet" -e MINER_ADDRESS="t27eWDgjFYJGVXmzrXeVjnb5J3uXDM9xH9v" -p 18232:18232 zfnd/zebra:latest +docker run --name zebra_local -e NETWORK="Testnet" -e MINER_ADDRESS="t27eWDgjFYJGVXmzrXeVjnb5J3uXDM9xH9v" -e ZEBRA_RPC_PORT=18232 -p 18232:18232 zfnd/zebra:latest ``` will start a container on Testnet and bind port 18232 on your Docker host, which @@ -42,3 +45,21 @@ is the standard Testnet RPC port. Notice that we also used a different rewards address. It starts with the prefix `t2`, indicating that it is a Testnet address. A Mainnet address would prevent Zebra from starting on Testnet, and conversely, a Testnet address would prevent Zebra from starting on Mainnet. + +To connect to the RPC port, you will need the contents of the [cookie +file](https://zebra.zfnd.org/user/mining.html?highlight=cookie#testing-the-setup) +Zebra uses for authentication. By default, it is stored at +`/home/zebra/.cache/zebra/.cookie`. You can print its contents by running + +```bash +docker exec -it zebra_local cat /home/zebra/.cache/zebra/.cookie +``` + +If you want to avoid authentication, you can turn it off by setting + +```toml +[rpc] +enable_cookie_auth = false +``` + +in Zebra's config file before you start the container. From 8b5a1a5e2b46dfffbdfe1b056710e3d307fbcc08 Mon Sep 17 00:00:00 2001 From: Marek Date: Fri, 14 Feb 2025 12:53:40 +0100 Subject: [PATCH 03/20] Revert "Rm `.env` files" This reverts commit caaa4559c3eac58173126eddd9f37b9a8642ad26. --- docker/.env | 33 +++++++++++++++++++++++++++ docker/test.env | 60 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 docker/.env create mode 100644 docker/test.env diff --git a/docker/.env b/docker/.env new file mode 100644 index 00000000000..2d96240f23e --- /dev/null +++ b/docker/.env @@ -0,0 +1,33 @@ +RUST_LOG=info +# This variable forces the use of color in the logs +ZEBRA_FORCE_USE_COLOR=1 +LOG_COLOR=true + +### +# Configuration Variables +# These variables are used to configure the zebra node +# Check the entrypoint.sh script for more details +### + +# The config file full path used in the Dockerfile. +ZEBRA_CONF_PATH=/etc/zebrad/zebrad.toml +# [network] +NETWORK=Mainnet +ZEBRA_LISTEN_ADDR=0.0.0.0 +# [consensus] +ZEBRA_CHECKPOINT_SYNC=true +# [state] +# Set this to change the default cached state directory +ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache +LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache +# [metrics] +METRICS_ENDPOINT_ADDR=0.0.0.0 +METRICS_ENDPOINT_PORT=9999 +# [tracing] +TRACING_ENDPOINT_ADDR=0.0.0.0 +TRACING_ENDPOINT_PORT=3000 +# [rpc] +RPC_LISTEN_ADDR=0.0.0.0 +# if ${RPC_PORT} is not set, it will use the default value for the current network +RPC_PORT=8232 + diff --git a/docker/test.env b/docker/test.env new file mode 100644 index 00000000000..fd2a7c876b7 --- /dev/null +++ b/docker/test.env @@ -0,0 +1,60 @@ +### +# Configuration Variables +# These variables are used to configure the zebra node +# Check the entrypoint.sh script for more details +### + +# Set this to change the default log level (must be set at build time) +RUST_LOG=info +# This variable forces the use of color in the logs +ZEBRA_FORCE_USE_COLOR=1 +LOG_COLOR=true +# Path to the config file. This variable has a default set in entrypoint.sh +# ZEBRA_CONF_PATH=/etc/zebrad/zebrad.toml +# [network] +NETWORK=Mainnet +# [state] +# Set this to change the default cached state directory +ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache +LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache +# [tracing] +LOG_COLOR=false +TRACING_ENDPOINT_ADDR=0.0.0.0 +TRACING_ENDPOINT_PORT=3000 + +#### +# Test Variables +# These variables are used to run tests in the Dockerfile +# Check the entrypoint.sh script for more details +#### + +# Unit tests +# TODO: These variables are evaluated to any value, even setting a NULL value will evaluate to true +# TEST_FAKE_ACTIVATION_HEIGHTS= +# ZEBRA_SKIP_NETWORK_TESTS +# ZEBRA_SKIP_IPV6_TESTS +RUN_ALL_TESTS= +RUN_ALL_EXPERIMENTAL_TESTS= +TEST_ZEBRA_EMPTY_SYNC= +ZEBRA_TEST_LIGHTWALLETD= +# Integration Tests +# Most of these tests require a cached state directory to save the network state +TEST_DISK_REBUILD= +# These tests needs a Zebra cached state +TEST_CHECKPOINT_SYNC= +GENERATE_CHECKPOINTS_MAINNET= +GENERATE_CHECKPOINTS_TESTNET= +TEST_UPDATE_SYNC= +# These tests need a Lightwalletd binary + a Zebra cached state +TEST_LWD_RPC_CALL= +TEST_GET_BLOCK_TEMPLATE= +TEST_SUBMIT_BLOCK= +# These tests need a Lightwalletd binary + Lightwalletd cached state + a Zebra cached state +TEST_LWD_UPDATE_SYNC= +TEST_LWD_GRPC= +TEST_LWD_TRANSACTIONS= +# Full sync tests +# These tests could take a long time to run, depending on the network +FULL_SYNC_MAINNET_TIMEOUT_MINUTES= +FULL_SYNC_TESTNET_TIMEOUT_MINUTES= +TEST_LWD_FULL_SYNC= From 814ed2b7821f37047353cb1b79352110c19dd5c2 Mon Sep 17 00:00:00 2001 From: Marek Date: Mon, 17 Feb 2025 10:11:41 +0100 Subject: [PATCH 04/20] Add `enable_cookie_auth` to default Zebra conf --- docker/default_zebra_config.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/default_zebra_config.toml b/docker/default_zebra_config.toml index d31a702ade4..bb5c17ced87 100644 --- a/docker/default_zebra_config.toml +++ b/docker/default_zebra_config.toml @@ -25,6 +25,11 @@ cache_dir = "/home/zebra/.cache/zebra" cookie_dir = "/home/zebra/.cache/zebra" +# To disable cookie authentication, uncomment the line below and set the value +# to false. + +# enable_cookie_auth = true + [state] cache_dir = "/home/zebra/.cache/zebra" From cec97662513f4315b3ce8f710d75c5645d0f7cf3 Mon Sep 17 00:00:00 2001 From: Marek Date: Tue, 18 Feb 2025 15:19:05 +0100 Subject: [PATCH 05/20] Rename `default_zebra_config.toml` --- .dockerignore | 2 +- book/src/user/mining-docker.md | 2 +- docker/Dockerfile | 4 ++-- ...default_zebra_config.toml => default-zebra-config.toml} | 7 ++++--- 4 files changed, 8 insertions(+), 7 deletions(-) rename docker/{default_zebra_config.toml => default-zebra-config.toml} (87%) diff --git a/.dockerignore b/.dockerignore index b26a7d4f413..137c05f2931 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,4 +21,4 @@ !zebra-* !zebrad !docker/entrypoint.sh -!docker/default_zebra_config.toml +!docker/default-zebra-config.toml diff --git a/book/src/user/mining-docker.md b/book/src/user/mining-docker.md index a65a32002ea..2550ba23610 100644 --- a/book/src/user/mining-docker.md +++ b/book/src/user/mining-docker.md @@ -28,7 +28,7 @@ find more info here https://docs.docker.com/engine/reference/commandline/run/#env. If you don't want to set any environment variables, you can edit the -`docker/default_zebra_config.toml` file, and pass it to Zebra before starting +`docker/default-zebra-config.toml` file, and pass it to Zebra before starting the container. There's an example in `docker/docker-compose.yml` of how to do that. diff --git a/docker/Dockerfile b/docker/Dockerfile index 320ff15aec4..0868ef98917 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -112,7 +112,7 @@ COPY --from=electriccoinco/lightwalletd:latest /usr/local/bin/lightwalletd /usr/ # Use the same default config as in the production environment. ENV ZEBRA_CONF_PATH="${HOME}/.config/zebrad.toml" -COPY --chown=${UID}:${GID} ./docker/default_zebra_config.toml ${ZEBRA_CONF_PATH} +COPY --chown=${UID}:${GID} ./docker/default-zebra-config.toml ${ZEBRA_CONF_PATH} ARG LWD_CACHE_DIR ENV LWD_CACHE_DIR="${HOME}/.cache/lwd" @@ -204,7 +204,7 @@ RUN adduser --system --gid ${GID} --uid ${UID} --home ${HOME} ${USER} ARG ZEBRA_CONF_PATH="${HOME}/.config/zebrad.toml" ENV ZEBRA_CONF_PATH=${ZEBRA_CONF_PATH} -COPY --chown=${UID}:${GID} ./docker/default_zebra_config.toml ${ZEBRA_CONF_PATH} +COPY --chown=${UID}:${GID} ./docker/default-zebra-config.toml ${ZEBRA_CONF_PATH} ARG ZEBRA_CACHE_DIR="${HOME}/.cache/zebra" ENV ZEBRA_CACHE_DIR=${ZEBRA_CACHE_DIR} diff --git a/docker/default_zebra_config.toml b/docker/default-zebra-config.toml similarity index 87% rename from docker/default_zebra_config.toml rename to docker/default-zebra-config.toml index bb5c17ced87..5b5e5b6ebd1 100644 --- a/docker/default_zebra_config.toml +++ b/docker/default-zebra-config.toml @@ -34,9 +34,10 @@ cookie_dir = "/home/zebra/.cache/zebra" cache_dir = "/home/zebra/.cache/zebra" [tracing] -# Zebra uses colored output if it is attached to a terminal. To disable colors, -# set `use_color` to false. To enable colors even for non-terminal outputs, set -# `use_color` to `true` and uncomment the line below. +# Zebra recognizes whether its logs are being written to a terminal or a file, +# and uses colored logs for terminals and uncolored logs for files. To force +# colors even for files, uncomment the line below. To disable colors, set +# `use_color` to false. # force_use_color = true use_color = true From 18f8efea50ec75d821be1f81b4a05cb8698fb12b Mon Sep 17 00:00:00 2001 From: Marek Date: Tue, 18 Feb 2025 15:19:59 +0100 Subject: [PATCH 06/20] fmt `prometheus.yaml` --- prometheus.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/prometheus.yaml b/prometheus.yaml index 5501da2b3f1..86ca1d5e7e4 100644 --- a/prometheus.yaml +++ b/prometheus.yaml @@ -1,7 +1,6 @@ scrape_configs: - - job_name: 'zebrad' + - job_name: "zebrad" scrape_interval: 500ms - metrics_path: '/' + metrics_path: "/" static_configs: - - targets: ['localhost:9999'] - + - targets: ["localhost:9999"] From 054c5a68279ebe70e0898b9591496a4d3b0405eb Mon Sep 17 00:00:00 2001 From: Marek Date: Tue, 18 Feb 2025 15:20:25 +0100 Subject: [PATCH 07/20] Update `docker/test.env` --- docker/test.env | 110 +++++++++++++++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 38 deletions(-) diff --git a/docker/test.env b/docker/test.env index fd2a7c876b7..c129a72a979 100644 --- a/docker/test.env +++ b/docker/test.env @@ -1,60 +1,94 @@ -### -# Configuration Variables -# These variables are used to configure the zebra node -# Check the entrypoint.sh script for more details -### - -# Set this to change the default log level (must be set at build time) -RUST_LOG=info -# This variable forces the use of color in the logs -ZEBRA_FORCE_USE_COLOR=1 -LOG_COLOR=true -# Path to the config file. This variable has a default set in entrypoint.sh -# ZEBRA_CONF_PATH=/etc/zebrad/zebrad.toml -# [network] -NETWORK=Mainnet -# [state] -# Set this to change the default cached state directory -ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache -# [tracing] -LOG_COLOR=false -TRACING_ENDPOINT_ADDR=0.0.0.0 -TRACING_ENDPOINT_PORT=3000 - -#### -# Test Variables -# These variables are used to run tests in the Dockerfile -# Check the entrypoint.sh script for more details -#### +# Configuration variables for running Zebra in Docker + +# Sets the network Zebra runs will run on. +# +# NETWORK=Mainnet + +# Zebra's RPC server is disabled by default. To enable it, set its port number. +# +# ZEBRA_RPC_PORT=8232 # Default RPC port number on Mainnet. +# ZEBRA_RPC_PORT=18323 # Default RPC port number on Testnet. + +# To disable cookie authentication, set the value below to false. +# +# ENABLE_COOKIE_AUTH=true + +# Sets a custom directory for the state and network caches. Zebra will also +# store its cookie authentication file in this directory. +# +# ZEBRA_CACHE_DIR="/home/zebra/.cache/zebra" + +# Sets custom Cargo features. Available features are listed at +# . +# +# Must be set at build time. +# +# FEATURES="" + +# Logging to a file is disabled by default. To enable it, uncomment the line +# below and alternatively set your own path. +# +# LOG_FILE="/home/zebra/.local/state/zebrad.log" + +# Zebra recognizes whether its logs are being written to a terminal or a file, +# and uses colored logs for terminals and uncolored logs for files. Setting the +# variable below to true will force colored logs even for files and setting it +# to false will disable colors even for terminals. +# +# LOG_COLOR=true + +# To disable logging to journald, set the value to false. +# +# USE_JOURNALD=true + +# If you are going to use Zebra as a backend for a mining pool, set your mining +# address. +# +# MINER_ADDRESS="your_mining_address" + +# Controls the output of `env_logger`: +# https://docs.rs/env_logger/latest/env_logger/ +# +# Must be set at build time. +# +# RUST_LOG=info # Unit tests -# TODO: These variables are evaluated to any value, even setting a NULL value will evaluate to true + +# TODO: These variables are evaluated to any value, even setting a NULL value +# will evaluate to true. +# # TEST_FAKE_ACTIVATION_HEIGHTS= -# ZEBRA_SKIP_NETWORK_TESTS -# ZEBRA_SKIP_IPV6_TESTS +# ZEBRA_SKIP_NETWORK_TESTS= +# ZEBRA_SKIP_IPV6_TESTS= RUN_ALL_TESTS= -RUN_ALL_EXPERIMENTAL_TESTS= TEST_ZEBRA_EMPTY_SYNC= ZEBRA_TEST_LIGHTWALLETD= + # Integration Tests -# Most of these tests require a cached state directory to save the network state + +# Most of these tests require a cached state directory to save the network state. TEST_DISK_REBUILD= -# These tests needs a Zebra cached state TEST_CHECKPOINT_SYNC= GENERATE_CHECKPOINTS_MAINNET= GENERATE_CHECKPOINTS_TESTNET= TEST_UPDATE_SYNC= -# These tests need a Lightwalletd binary + a Zebra cached state +TEST_SCANNER= + +# These tests need a Lightwalletd binary + a Zebra cached state. TEST_LWD_RPC_CALL= TEST_GET_BLOCK_TEMPLATE= TEST_SUBMIT_BLOCK= -# These tests need a Lightwalletd binary + Lightwalletd cached state + a Zebra cached state + +# These tests need a Lightwalletd binary + Lightwalletd cached state + a Zebra +# cached state. TEST_LWD_UPDATE_SYNC= TEST_LWD_GRPC= TEST_LWD_TRANSACTIONS= + # Full sync tests -# These tests could take a long time to run, depending on the network + +# These tests take 3 days on Mainnet and one day on Testnet. FULL_SYNC_MAINNET_TIMEOUT_MINUTES= FULL_SYNC_TESTNET_TIMEOUT_MINUTES= TEST_LWD_FULL_SYNC= From ff2addf8bec3396c602488080ebd4dfdb0008390 Mon Sep 17 00:00:00 2001 From: Marek Date: Tue, 18 Feb 2025 15:24:16 +0100 Subject: [PATCH 08/20] Update `docker/.env` --- docker/.env | 85 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 32 deletions(-) diff --git a/docker/.env b/docker/.env index 2d96240f23e..b4e8bade406 100644 --- a/docker/.env +++ b/docker/.env @@ -1,33 +1,54 @@ -RUST_LOG=info -# This variable forces the use of color in the logs -ZEBRA_FORCE_USE_COLOR=1 -LOG_COLOR=true - -### -# Configuration Variables -# These variables are used to configure the zebra node -# Check the entrypoint.sh script for more details -### - -# The config file full path used in the Dockerfile. -ZEBRA_CONF_PATH=/etc/zebrad/zebrad.toml -# [network] -NETWORK=Mainnet -ZEBRA_LISTEN_ADDR=0.0.0.0 -# [consensus] -ZEBRA_CHECKPOINT_SYNC=true -# [state] -# Set this to change the default cached state directory -ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache -# [metrics] -METRICS_ENDPOINT_ADDR=0.0.0.0 -METRICS_ENDPOINT_PORT=9999 -# [tracing] -TRACING_ENDPOINT_ADDR=0.0.0.0 -TRACING_ENDPOINT_PORT=3000 -# [rpc] -RPC_LISTEN_ADDR=0.0.0.0 -# if ${RPC_PORT} is not set, it will use the default value for the current network -RPC_PORT=8232 +# Configuration variables for running Zebra in Docker +# Sets the network Zebra runs will run on. +# +# NETWORK=Mainnet + +# Zebra's RPC server is disabled by default. To enable it, set its port number. +# +# ZEBRA_RPC_PORT=8232 # Default RPC port number on Mainnet. +# ZEBRA_RPC_PORT=18323 # Default RPC port number on Testnet. + +# To disable cookie authentication, set the value below to false. +# +# ENABLE_COOKIE_AUTH=true + +# Sets a custom directory for the state and network caches. Zebra will also +# store its cookie authentication file in this directory. +# +# ZEBRA_CACHE_DIR="/home/zebra/.cache/zebra" + +# Sets custom Cargo features. Available features are listed at +# . +# +# Must be set at build time. +# +# FEATURES="" + +# Logging to a file is disabled by default. To enable it, uncomment the line +# below and alternatively set your own path. +# +# LOG_FILE="/home/zebra/.local/state/zebrad.log" + +# Zebra recognizes whether its logs are being written to a terminal or a file, +# and uses colored logs for terminals and uncolored logs for files. Setting the +# variable below to true will force colored logs even for files and setting it +# to false will disable colors even for terminals. +# +# LOG_COLOR=true + +# To disable logging to journald, set the value to false. +# +# USE_JOURNALD=true + +# If you are going to use Zebra as a backend for a mining pool, set your mining +# address. +# +# MINER_ADDRESS="your_mining_address" + +# Controls the output of `env_logger`: +# https://docs.rs/env_logger/latest/env_logger/ +# +# Must be set at build time. +# +# RUST_LOG=info From 6e9e28a39fc82d2235bccdabe7af8a208cac245c Mon Sep 17 00:00:00 2001 From: Marek Date: Tue, 18 Feb 2025 15:24:40 +0100 Subject: [PATCH 09/20] Refactor `docker compose` for lwd --- docker/docker-compose.lwd.yml | 44 ++++++++++++++++++++--------------- docker/zcash.conf | 2 ++ 2 files changed, 27 insertions(+), 19 deletions(-) create mode 100644 docker/zcash.conf diff --git a/docker/docker-compose.lwd.yml b/docker/docker-compose.lwd.yml index 7d8c56b1855..456e7602d97 100644 --- a/docker/docker-compose.lwd.yml +++ b/docker/docker-compose.lwd.yml @@ -1,15 +1,22 @@ -version: "3.8" - services: zebra: + container_name: zebra + image: zfnd/zebra + platform: linux/amd64 + restart: unless-stopped + deploy: + resources: + reservations: + cpus: "4" + memory: 16G + volumes: + - zebrad-cache:/home/zebra/.cache/zebra + tty: true + environment: + - ZEBRA_RPC_PORT=8232 + - ENABLE_COOKIE_AUTH=false ports: - - "8232:8232" # Opens an RPC endpoint (for lightwalletd and mining) - healthcheck: - start_period: 1m - interval: 15s - timeout: 10s - retries: 3 - test: ["CMD-SHELL", "curl --data-binary '{\"id\":\"curltest\", \"method\": \"getinfo\"}' -H 'content-type: application/json' 127.0.0.1:8232 || exit 1"] + - "8232:8232" lightwalletd: image: electriccoinco/lightwalletd @@ -29,13 +36,11 @@ services: configs: - source: lwd_config target: /etc/lightwalletd/zcash.conf - uid: '2002' # Golang's container default user uid - gid: '2002' # Golang's container default group gid - mode: 0440 volumes: - - litewalletd-data:/var/lib/lightwalletd/db - #! This setup with --no-tls-very-insecure is only for testing purposes - #! For production environments follow the guidelines here: https://github.com/zcash/lightwalletd#production-usage + - lwd-cache:/var/lib/lightwalletd/db + #! This setup with `--no-tls-very-insecure` is only for testing purposes. + #! For production environments, follow the guidelines here: + #! https://github.com/zcash/lightwalletd#production-usage command: > --no-tls-very-insecure --grpc-bind-addr=0.0.0.0:9067 @@ -50,10 +55,11 @@ services: configs: lwd_config: - # Change the following line to point to a zcash.conf on your host machine - # to allow for easy configuration changes without rebuilding the image - file: ./zcash-lightwalletd/zcash.conf + file: ./zcash.conf volumes: - litewalletd-data: + zebrad-cache: + driver: local + + lwd-cache: driver: local diff --git a/docker/zcash.conf b/docker/zcash.conf new file mode 100644 index 00000000000..22f9ab8495d --- /dev/null +++ b/docker/zcash.conf @@ -0,0 +1,2 @@ +rpcpassword=none +rpcbind=zebra From 875a4c768a24bb831aa03bc990ccbff8653c8ff5 Mon Sep 17 00:00:00 2001 From: Marek Date: Tue, 18 Feb 2025 15:25:53 +0100 Subject: [PATCH 10/20] Enable disabling cookie authentication --- docker/entrypoint.sh | 55 ++++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index f58d7c57147..e10d263c1e2 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -6,7 +6,7 @@ # # ## Notes # -# - `$ZEBRA_CONF_PATH` must point to a Zebra conf file writable by `$USER`. +# - `$ZEBRA_CONF_PATH` must point to a Zebra conf file. set -eo pipefail @@ -16,25 +16,32 @@ if [[ ! -f "${ZEBRA_CONF_PATH}" ]]; then exit 1 fi -# Populates the config file for Zebra, using the env vars set by the Dockerfile -# or user. -# -# Also prints the content of the generated config file. +# Generates a config file for Zebra using env vars set in "docker/.env" and +# prints the location of the generated config file. # # ## Positional Parameters # -# - "$1": the file to write the config to +# - "$1": the file to read the default config from prepare_conf_file() { - # Set a custom `network`. + # Copy the default config to a new location for writing. + CONF=~/zebrad.toml + cp "${1}" "${CONF}" + + # Set a custom network. if [[ "${NETWORK}" ]]; then - sed -i '/network = ".*"/s/".*"/"'"${NETWORK//\"/}"'"/' "${1}" + sed -i '/network = ".*"/s/".*"/"'"${NETWORK//\"/}"'"/' "${CONF}" fi # Enable the RPC server by setting its port. if [[ "${ZEBRA_RPC_PORT}" ]]; then - sed -i '/# listen_addr = "0.0.0.0:18232" # Testnet/d' "${1}" - sed -i 's/ *# Mainnet$//' "${1}" - sed -i '/# listen_addr = "0.0.0.0:8232"/s/^# //; s/8232/'"${ZEBRA_RPC_PORT//\"/}"'/' "${1}" + sed -i '/# listen_addr = "0.0.0.0:18232" # Testnet/d' "${CONF}" + sed -i 's/ *# Mainnet$//' "${CONF}" + sed -i '/# listen_addr = "0.0.0.0:8232"/s/^# //; s/8232/'"${ZEBRA_RPC_PORT//\"/}"'/' "${CONF}" + fi + + # Disable or enable cookie authentication. + if [[ "${ENABLE_COOKIE_AUTH}" ]]; then + sed -i '/# enable_cookie_auth = true/s/^# //; s/true/'"${ENABLE_COOKIE_AUTH//\"/}"'/' "${CONF}" fi # Set a custom state, network and cookie cache dirs. @@ -44,41 +51,40 @@ prepare_conf_file() { # use them to set the cache dirs separately if needed. if [[ "${ZEBRA_CACHE_DIR}" ]]; then mkdir -p "${ZEBRA_CACHE_DIR//\"/}" - sed -i 's|_dir = ".*"|_dir = "'"${ZEBRA_CACHE_DIR//\"/}"'"|' "${1}" + sed -i 's|_dir = ".*"|_dir = "'"${ZEBRA_CACHE_DIR//\"/}"'"|' "${CONF}" fi # Enable the Prometheus metrics endpoint. if [[ "${FEATURES}" == *"prometheus"* ]]; then - sed -i '/# endpoint_addr = "0.0.0.0:9999" # Prometheus/s/^# //' "${1}" + sed -i '/# endpoint_addr = "0.0.0.0:9999" # Prometheus/s/^# //' "${CONF}" fi # Enable logging to a file by setting a custom log file path. if [[ "${LOG_FILE}" ]]; then mkdir -p "$(dirname "${LOG_FILE//\"/}")" - sed -i 's|# log_file = ".*"|log_file = "'"${LOG_FILE//\"/}"'"|' "${1}" + sed -i 's|# log_file = ".*"|log_file = "'"${LOG_FILE//\"/}"'"|' "${CONF}" fi # Enable or disable colored logs. if [[ "${LOG_COLOR}" ]]; then - sed -i '/# force_use_color = true/s/^# //' "${1}" - sed -i '/use_color = true/s/true/'"${LOG_COLOR//\"/}"'/' "${1}" + sed -i '/# force_use_color = true/s/^# //' "${CONF}" + sed -i '/use_color = true/s/true/'"${LOG_COLOR//\"/}"'/' "${CONF}" fi # Enable or disable logging to systemd-journald. if [[ "${USE_JOURNALD}" ]]; then - sed -i '/# use_journald = true/s/^# //; s/true/'"${USE_JOURNALD//\"/}"'/' "${1}" + sed -i '/# use_journald = true/s/^# //; s/true/'"${USE_JOURNALD//\"/}"'/' "${CONF}" fi # Set a mining address. if [[ "${MINER_ADDRESS}" ]]; then - sed -i '/# miner_address = ".*"/{s/^# //; s/".*"/"'"${MINER_ADDRESS//\"/}"'"/}' "${1}" + sed -i '/# miner_address = ".*"/{s/^# //; s/".*"/"'"${MINER_ADDRESS//\"/}"'"/}' "${CONF}" fi # Trim all comments and empty lines. - sed -i '/^#/d; /^$/d' "${1}" + sed -i '/^#/d; /^$/d' "${CONF}" - echo "Prepared the following Zebra config:" - cat "$1" + echo "${CONF}" } # Checks if a directory contains subdirectories @@ -262,7 +268,10 @@ run_tests() { # Main Script Logic -prepare_conf_file "$ZEBRA_CONF_PATH" +echo "Prepared the following Zebra config:" + +CONF_PATH=$(prepare_conf_file "${ZEBRA_CONF_PATH}") +cat "${CONF_PATH}" # - If "$1" is "--", "-", or "zebrad", run `zebrad` with the remaining params. # - If "$1" is "tests", run tests. @@ -271,7 +280,7 @@ prepare_conf_file "$ZEBRA_CONF_PATH" case "$1" in --* | -* | zebrad) shift - exec zebrad --config "${ZEBRA_CONF_PATH}" "$@" + exec zebrad --config "${CONF_PATH}" "$@" ;; test) shift From ec8bf253fc330915670817164d75ca89c37f9deb Mon Sep 17 00:00:00 2001 From: Marek Date: Tue, 18 Feb 2025 15:26:09 +0100 Subject: [PATCH 11/20] Update `docker compose` for tests --- docker/docker-compose.test.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/docker/docker-compose.test.yml b/docker/docker-compose.test.yml index b5730359bcb..d3659612a82 100644 --- a/docker/docker-compose.test.yml +++ b/docker/docker-compose.test.yml @@ -1,30 +1,13 @@ -version: "3.8" - services: zebra: + container_name: zebra build: context: ../ dockerfile: docker/Dockerfile target: tests - restart: unless-stopped - deploy: - resources: - reservations: - cpus: "4" - memory: 16G - # Change this to the command you want to run, respecting the entrypoint.sh - # For example, to run the tests, use the following command: - # command: ["cargo", "test", "--locked", "--release", "--features", "${FEATURES}", "--package", "zebrad", "--test", "acceptance", "--", "--nocapture", "--include-ignored", "sync_large_checkpoints_"] volumes: - zebrad-cache:/home/zebra/.cache/zebra - lwd-cache:/home/zebra/.cache/lwd - ports: - # Zebra uses the following inbound and outbound TCP ports - - "8232:8232" # Opens an RPC endpoint (for wallet storing and mining) - - "8233:8233" # Mainnet Network (for peer connections) - - "18233:18233" # Testnet Network - # - "9999:9999" # Metrics - # - "3000:3000" # Tracing env_file: - test.env From 4f921cf6162fb47568f9f2740fea291cc948174e Mon Sep 17 00:00:00 2001 From: Marek Date: Tue, 18 Feb 2025 15:26:22 +0100 Subject: [PATCH 12/20] Update general `docker compose` --- docker/docker-compose.yml | 53 ++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index f9c71d6e0d4..a30e248b020 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,13 +1,8 @@ -version: "3.8" - services: zebra: + container_name: zebra image: zfnd/zebra platform: linux/amd64 - build: - context: ../ - dockerfile: docker/Dockerfile - target: runtime restart: unless-stopped deploy: resources: @@ -16,33 +11,33 @@ services: memory: 16G env_file: - .env - logging: - options: - max-size: "10m" - max-file: "5" - #! Uncomment the `configs` mapping below to use the `zebrad.toml` config file from the host machine - #! NOTE: This will override the zebrad.toml in the image and make some variables irrelevant - # configs: - # - source: zebra_config - # target: /etc/zebrad/zebrad.toml - # uid: '2001' # Rust's container default user uid - # gid: '2001' # Rust's container default group gid - # mode: 0440 volumes: - zebrad-cache:/home/zebra/.cache/zebra - ports: - # Zebra uses the following default inbound and outbound TCP ports - - "8233:8233" # Mainnet Network (for peer connections) - # - "8232:8232" # Opens an RPC endpoint (for wallet storing and mining) - # - "18233:18233" # Testnet Network - # - "9999:9999" # Metrics - # - "3000:3000" # Tracing + # Having `tty` set to true makes Zebra use colored logs. + tty: true + #! Uncomment the `configs` mapping below to make your custom configuration + #! take effect. + # + # configs: + # - source: zebra-config + # target: /home/zebra/.config/zebrad.toml + # + # Uncomment the `ports` mapping below to map ports between the container and + # host. + # + # ports: + # - "8232:8232" # RPC endpoint on Mainnet + # - "18232:18232" # RPC endpoint on Testnet + # - "8233:8233" # peer connections on Mainnet + # - "18233:18233" # peer connections on Testnet + # - "9999:9999" # Metrics + # - "3000:3000" # Tracing configs: - zebra_config: - # Change the following line to point to a zebrad.toml on your host machine - # to allow for easy configuration changes without rebuilding the image - file: ../zebrad/tests/common/configs/v1.0.0-rc.2.toml + zebra-config: + #! To customize the default configuration, edit this file before starting + #! the container. + file: ./default-zebra-config.toml volumes: zebrad-cache: From cf3ee9fdd53d0bde09befce4fab696deb855bb4e Mon Sep 17 00:00:00 2001 From: Marek Date: Tue, 18 Feb 2025 15:26:34 +0100 Subject: [PATCH 13/20] Update docs for running Zebra in Docker --- book/src/user/docker.md | 227 +++++++++++++++++++--------------------- 1 file changed, 105 insertions(+), 122 deletions(-) diff --git a/book/src/user/docker.md b/book/src/user/docker.md index 7709bdca4e7..db235b056ac 100644 --- a/book/src/user/docker.md +++ b/book/src/user/docker.md @@ -1,176 +1,159 @@ # Zebra with Docker -The easiest way to run Zebra is using [Docker](https://docs.docker.com/get-docker/). +The foundation maintains a Docker infrastructure for deploying and testing Zebra. -We've embraced Docker in Zebra for most of the solution lifecycle, from development environments to CI (in our pipelines), and deployment to end users. +## Quick Start -> [!TIP] -> We recommend using `docker compose` sub-command over the plain `docker` CLI, especially for more advanced use-cases like running CI locally, as it provides a more convenient and powerful way to manage multi-container Docker applications. See [CI/CD Local Testing](#cicd-local-testing) for more information, and other compose files available in the [docker](https://github.com/ZcashFoundation/zebra/tree/main/docker) folder. - -## Quick usage - -You can deploy Zebra for daily use with the images available in [Docker Hub](https://hub.docker.com/r/zfnd/zebra) or build it locally for testing. - -### Ready to use image - -Using `docker compose`: +To get Zebra quickly up and running, you can use an off-the-rack image from +[Docker Hub](https://hub.docker.com/r/zfnd/zebra/tags): ```shell -docker compose -f docker/docker-compose.yml up +docker run --name zebra zfnd/zebra ``` -With plain `docker` CLI: +If you want to preserve Zebra's state, you can create a Docker volume: ```shell docker volume create zebrad-cache - -docker run -d --platform linux/amd64 \ - --restart unless-stopped \ - --env-file .env \ - --mount type=volume,source=zebrad-cache,target=/home/zebra/.cache/zebra \ - -p 8233:8233 \ - --memory 16G \ - --cpus 4 \ - zfnd/zebra ``` -### Build it locally +And mount it before you start the container: ```shell -git clone --depth 1 --branch v2.2.0 https://github.com/ZcashFoundation/zebra.git -docker build --file docker/Dockerfile --target runtime --tag zebra:local . -docker run --detach zebra:local -``` - -### Alternatives - -See [Building Zebra](https://github.com/ZcashFoundation/zebra#building-zebra) for more information. - -## Advanced usage - -You're able to specify various parameters when building or launching the Docker image, which are meant to be used by developers and CI pipelines. For example, specifying the Network where Zebra will run (Mainnet, Testnet, etc), or enabling features like metrics with Prometheus. - -For example, if we'd like to enable metrics on the image, we'd build it using the following `build-arg`: - -> [!IMPORTANT] -> To fully use and display the metrics, you'll need to run a Prometheus and Grafana server, and configure it to scrape and visualize the metrics endpoint. This is explained in more detailed in the [Metrics](https://zebra.zfnd.org/user/metrics.html#zebra-metrics) section of the User Guide. - -```shell -docker build -f ./docker/Dockerfile --target runtime --build-arg FEATURES='default-release-binaries prometheus' --tag local/zebra.mining:latest . +docker run \ + --mount type=volume,source=zebrad-cache,target=/home/zebra/.cache/zebra \ + --name zebra + zfnd/zebra ``` -To increase the log output we can optionally add these `build-arg`s: +You can also use `docker compose`, which we recommend. First get the repo: ```shell ---build-arg RUST_BACKTRACE=full --build-arg RUST_LOG=debug --build-arg COLORBT_SHOW_HIDDEN=1 +git clone --depth 1 --branch v2.2.0 https://github.com/ZcashFoundation/zebra.git +cd zebra ``` -And after our image has been built, we can run it on `Mainnet` with the following command, which will expose the metrics endpoint on port `9999` and force the logs to be colored: +Then run: ```shell -docker run --env LOG_COLOR="true" -p 9999:9999 local/zebra.mining -``` - -Based on our actual `entrypoint.sh` script, the following configuration file will be generated (on the fly, at startup) and used by Zebra: - -```toml -[network] -network = "Mainnet" -listen_addr = "0.0.0.0" -[state] -cache_dir = "/home/zebra/.cache/zebra" -[metrics] -endpoint_addr = "127.0.0.1:9999" +docker compose -f docker/docker-compose.yml up ``` -### Running Zebra with Lightwalletd +## Custom Images -To run Zebra with Lightwalletd, we recommend using the provided `docker compose` files for Zebra and Lightwalletd, which will start both services and connect them together, while exposing ports, mounting volumes, and setting environment variables. +If you want to use your own images with, for example, some opt-in compilation +features enabled, add the desired features to the `FEATURES` variable in the +`docker/.env` file and build the image: ```shell -docker compose -f docker/docker-compose.yml -f docker/docker-compose.lwd.yml up +docker build \ + --file docker/Dockerfile \ + --env-file docker/.env \ + --target runtime \ + --tag zebra:local \ + . ``` -### CI/CD Local Testing - -To run CI tests locally, which mimics the testing done in our CI pipelines on GitHub Actions, use the `docker-compose.test.yml` file. This setup allows for a consistent testing environment both locally and in CI. - -#### Running Tests Locally - -1. **Setting Environment Variables**: - - Modify the `test.env` file to set the desired test configurations. - - For running all tests, set `RUN_ALL_TESTS=1` in `test.env`. - -2. **Starting the Test Environment**: - - Use Docker Compose to start the testing environment: - - ```shell - docker-compose -f docker/docker-compose.test.yml up - ``` - - - This will start the Docker container and run the tests based on `test.env` settings. - -3. **Viewing Test Output**: - - The test results and logs will be displayed in the terminal. +All available Cargo features are listed at +. -4. **Stopping the Environment**: - - Once testing is complete, stop the environment using: +## Configuring Zebra - ```shell - docker-compose -f docker/docker-compose.test.yml down - ``` +To configure Zebra, edit the `docker/default-zebra-config.toml` config file and +uncomment the `configs` mapping in `docker/docker-compose.yml` so your config +takes effect. You can see if your config works as intended by looking at Zebra's +logs. -This approach ensures you can run the same tests locally that are run in CI, providing a robust way to validate changes before pushing to the repository. +Alternatively, you can configure Zebra by setting the environment variables in +the `docker/.env` file. Note that the config options of this file are limited to +the variables already present in the commented out blocks in it and adding new +ones will not be effective. Also note that the values of the variables take +precedence over the values set in the `docker/default-zebra-config.toml` config +file. The `docker/.env` file serves as a quick way to override the most commonly +used settings for Zebra, whereas the `docker/default-zebra-config.toml` file +provides full config capabilities. -### Build and Run Time Configuration +### RPC -#### Build Time Arguments +Zebra's RPC server is disabled by default. To enable it, you need to set its RPC +port. You can do that either in the `docker/default-zebra-config.toml` file or +`docker/.env` file, as described in the two paragraphs above. -#### Configuration +When connecting to Zebra's RPC server, your RPC clients need to provide an +authentication cookie to the server or you need to turn the authentication off +in Zebra's config. By default, the cookie file is stored at +`/home/zebra/.cache/zebra/.cookie` in the container. You can print its contents +by running -- `FEATURES`: Specifies the features to build `zebrad` with. Example: `"default-release-binaries getblocktemplate-rpcs"` - -#### Logging +```bash +docker exec zebra cat /home/zebra/.cache/zebra/.cookie +``` -- `RUST_LOG`: Sets the trace log level. Example: `"debug"` -- `RUST_BACKTRACE`: Enables or disables backtraces. Example: `"full"` -- `RUST_LIB_BACKTRACE`: Enables or disables library backtraces. Example: `1` -- `COLORBT_SHOW_HIDDEN`: Enables or disables showing hidden backtraces. Example: `1` +when the `zebra` container is running. Note that Zebra generates the cookie file +only if the RPC server is enabled, and each Zebra instance generates a unique +one. To turn the authentication off, either set `ENABLE_COOKIE_AUTH=false` in +`docker/.env` or set -#### Tests +```toml +[rpc] +enable_cookie_auth = false +``` -- `ZEBRA_SKIP_IPV6_TESTS`: Skips IPv6 tests. Example: `1` +in `docker/default-zebra-config.toml` and mount this config file into the +container's filesystem in `docker/docker-compose.yml` as described at the +beginning of this section. -#### CI/CD +## Examples -- `SHORT_SHA`: Represents the short SHA of the commit. Example: `"a1b2c3d"` +To make the initial setup of Zebra with other services easier, we provide some +example files for `docker compose`. The following subsections will walk you +through those examples. -#### Run Time Variables +### Running Zebra with Lightwalletd -- `NETWORK`: Specifies the network type. Example: `"Mainnet"` +The following command will run Zebra with Lightwalletd: -#### Zebra Configuration +```shell +docker compose -f docker/docker-compose.lwd.yml up +``` -- `ZEBRA_CACHE_DIR`: Directory for cached state. Example: `"/home/zebra/.cache/zebra"` +Note that Docker will run Zebra with the RPC server enabled and the cookie +authentication mechanism disabled since Lightwalletd doesn't support it. Instead +of configuring Zebra via the recommended config file or `docker/.env` file, we +configured the RPC server by setting environment variables directly in the +`docker/docker-compose.lwd.yml` file. This is a shortcut we can take when we are +familiar with the `docker/.env` file. -#### Mining Configuration +### Running Zebra with Prometheus and Grafana -- `RPC_LISTEN_ADDR`: Address for RPC to listen on. Example: `"0.0.0.0"` -- `MINER_ADDRESS`: Address for the miner. Example: `"t1XhG6pT9xRqRQn3BHP7heUou1RuYrbcrCc"` +The following commands will run Zebra with Prometheus and Grafana: -#### Other Configuration +```shell +docker compose -f docker/docker-compose.grafana.yml build --no-cache +docker compose -f docker/docker-compose.grafana.yml up +``` -- `METRICS_ENDPOINT_ADDR`: Address for metrics endpoint. Example: `"0.0.0.0"` -- `METRICS_ENDPOINT_PORT`: Port for metrics endpoint. Example: `9999` -- `LOG_FILE`: Path to the log file. Example: `"/path/to/log/file.log"` -- `LOG_COLOR`: Enables or disables log color. Example: `false` -- `TRACING_ENDPOINT_ADDR`: Address for tracing endpoint. Example: `"0.0.0.0"` -- `TRACING_ENDPOINT_PORT`: Port for tracing endpoint. Example: `3000` +In this example, we build a local Zebra image with the `prometheus` Cargo +compilation feature. Note that we enable this feature by specifying its name in +the build arguments. Having this Cargo feature specified at build time makes +`cargo` compile Zebra with the metrics support for Prometheus enabled. Note that +we also specify this feature as an environment variable at run time. Having this +feature specified at run time makes Docker's entrypoint script configure Zebra +to open a scraping endpoint on `localhost:9999` for Prometheus. -Specific tests are defined in `docker/test.env` file and can be enabled by setting the corresponding environment variable to `1`. +Once all services are up, the Grafana web UI should be available at +`localhost:3000`, the Prometheus web UI should be at `localhost:9090`, and +Zebra's scraping page should be at `localhost:9999`. The default login and +password for Grafana are both `admin`. To make Grafana use Prometheus, you need +to add Prometheus as a data source with the URL `http://localhost:9090` in +Grafana's UI. You can then import various Grafana dashboards from the `grafana` +directory in the Zebra repo. -## Registries +### Running CI Tests Locally -The images built by the Zebra team are all publicly hosted. Old image versions meant to be used by our [CI pipeline](https://github.com/ZcashFoundation/zebra/blob/main/.github/workflows/ci-integration-tests-gcp.yml) (`zebrad-test`, `lightwalletd`) might be deleted on a scheduled basis. +To run CI tests locally, first set the variables in the `test.env` file to +configure the tests, then run: -We use [Docker Hub](https://hub.docker.com/r/zfnd/zebra) for end-user images and [Google Artifact Registry](https://console.cloud.google.com/artifacts/docker/zfnd-dev-zebra/us/zebra) to build external tools and test images. +```shell +docker-compose -f docker/docker-compose.test.yml up +``` From d7920af8fc9dbd7a602193832a775d5a38947f72 Mon Sep 17 00:00:00 2001 From: Marek Date: Tue, 18 Feb 2025 15:26:54 +0100 Subject: [PATCH 14/20] Add example `docker compose` file for Grafana --- docker/docker-compose.grafana.yml | 52 +++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 docker/docker-compose.grafana.yml diff --git a/docker/docker-compose.grafana.yml b/docker/docker-compose.grafana.yml new file mode 100644 index 00000000000..2c7b6b4d7ab --- /dev/null +++ b/docker/docker-compose.grafana.yml @@ -0,0 +1,52 @@ +services: + zebra: + container_name: zebra + build: + context: ../ + dockerfile: docker/Dockerfile + target: runtime + args: + - FEATURES=prometheus + volumes: + - zebrad-cache:/home/zebra/.cache/zebra + tty: true + environment: + - FEATURES=prometheus + network_mode: "host" + ports: + - 9999:9999 + + prometheus: + container_name: prometheus + image: prom/prometheus + volumes: + - prometheus-cache:/prometheus + configs: + - source: prometheus-config + target: /etc/prometheus/prometheus.yml + network_mode: "host" + ports: + - 9090:9090 + + grafana: + container_name: grafana + image: grafana/grafana + volumes: + - grafana-cache:/var/lib/grafana + network_mode: "host" + ports: + - 3000:3000 + +volumes: + zebrad-cache: + driver: local + + grafana-cache: + driver: local + + prometheus-cache: + driver: local + +configs: + prometheus-config: + file: ../prometheus.yaml From f9abcf27b6afbaf21b071abee3e5495e92d41c11 Mon Sep 17 00:00:00 2001 From: Marek Date: Tue, 18 Feb 2025 15:48:38 +0100 Subject: [PATCH 15/20] Fix a bug in an example command --- book/src/user/docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/book/src/user/docker.md b/book/src/user/docker.md index db235b056ac..1dbe9425bee 100644 --- a/book/src/user/docker.md +++ b/book/src/user/docker.md @@ -22,7 +22,7 @@ And mount it before you start the container: ```shell docker run \ --mount type=volume,source=zebrad-cache,target=/home/zebra/.cache/zebra \ - --name zebra + --name zebra \ zfnd/zebra ``` From fffde2a5bc3bf24c0194dd2a095c84887804bedc Mon Sep 17 00:00:00 2001 From: Marek Date: Tue, 18 Feb 2025 20:12:16 +0100 Subject: [PATCH 16/20] Refactor test execution logic in entrypoint --- docker/entrypoint.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index e10d263c1e2..eca56196418 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -257,12 +257,7 @@ run_tests() { run_cargo_test "${FEATURES}" "submit_block" else - if [[ "$1" == "zebrad" ]]; then - shift - exec zebrad -c "${ZEBRA_CONF_PATH}" "$@" - else - exec "$@" - fi + exec "$@" fi } @@ -274,7 +269,9 @@ CONF_PATH=$(prepare_conf_file "${ZEBRA_CONF_PATH}") cat "${CONF_PATH}" # - If "$1" is "--", "-", or "zebrad", run `zebrad` with the remaining params. -# - If "$1" is "tests", run tests. +# - If "$1" is "tests": +# - and "$2" is "zebrad", run `zebrad` with the remaining params, +# - else run tests with the remaining params. # - TODO: If "$1" is "monitoring", start a monitoring node. # - If "$1" doesn't match any of the above, run "$@" directly. case "$1" in @@ -284,7 +281,12 @@ case "$1" in ;; test) shift - run_tests "$@" + if [[ "$1" == "zebrad" ]]; then + shift + exec zebrad --config "${CONF_PATH}" "$@" + else + run_tests "$@" + fi ;; monitoring) # TODO: Impl logic for starting a monitoring node. From 151633812c62f8aac418adb365d2e4cc4e5d1469 Mon Sep 17 00:00:00 2001 From: Marek Date: Wed, 19 Feb 2025 12:28:43 +0100 Subject: [PATCH 17/20] Rename `v2.1.0.toml` conf to `custom-conf.toml` --- zebrad/tests/common/configs/custom-conf.toml | 54 +++++++++++++ zebrad/tests/common/configs/v2.1.0.toml | 85 -------------------- 2 files changed, 54 insertions(+), 85 deletions(-) create mode 100644 zebrad/tests/common/configs/custom-conf.toml delete mode 100644 zebrad/tests/common/configs/v2.1.0.toml diff --git a/zebrad/tests/common/configs/custom-conf.toml b/zebrad/tests/common/configs/custom-conf.toml new file mode 100644 index 00000000000..98b22ecdc0f --- /dev/null +++ b/zebrad/tests/common/configs/custom-conf.toml @@ -0,0 +1,54 @@ +[consensus] +checkpoint_sync = true + +[mempool] +eviction_memory_time = "1h" +tx_cost_limit = 80000000 + +[metrics] + +[mining] +debug_like_zcashd = true +extra_coinbase_data = "do you even shield?" + +[network] +cache_dir = true +crawl_new_peer_interval = "1m 1s" +initial_mainnet_peers = [ + "dnsseed.z.cash:8233", + "dnsseed.str4d.xyz:8233", + "mainnet.seeder.zfnd.org:8233", + "mainnet.is.yolo.money:8233", +] +initial_testnet_peers = [ + "dnsseed.testnet.z.cash:18233", + "testnet.seeder.zfnd.org:18233", + "testnet.is.yolo.money:18233", +] +listen_addr = "0.0.0.0:8233" +max_connections_per_ip = 1 +network = "Mainnet" +peerset_initial_target_size = 25 + +[rpc] +cookie_dir = "cache_dir" +debug_force_finished_sync = false +enable_cookie_auth = true +parallel_cpu_threads = 0 + +[state] +cache_dir = "cache_dir" +delete_old_database = true +ephemeral = false + +[sync] +checkpoint_verify_concurrency_limit = 1000 +download_concurrency_limit = 50 +full_verify_concurrency_limit = 20 +parallel_cpu_threads = 0 + +[tracing] +buffer_limit = 128000 +force_use_color = false +use_color = true +use_journald = false diff --git a/zebrad/tests/common/configs/v2.1.0.toml b/zebrad/tests/common/configs/v2.1.0.toml deleted file mode 100644 index 54613ba842a..00000000000 --- a/zebrad/tests/common/configs/v2.1.0.toml +++ /dev/null @@ -1,85 +0,0 @@ -# Default configuration for zebrad. -# -# This file can be used as a skeleton for custom configs. -# -# Unspecified fields use default values. Optional fields are Some(field) if the -# field is present and None if it is absent. -# -# This file is generated as an example using zebrad's current defaults. -# You should set only the config options you want to keep, and delete the rest. -# Only a subset of fields are present in the skeleton, since optional values -# whose default is None are omitted. -# -# The config format (including a complete list of sections and fields) is -# documented here: -# https://docs.rs/zebrad/latest/zebrad/config/struct.ZebradConfig.html -# -# zebrad attempts to load configs in the following order: -# -# 1. The -c flag on the command line, e.g., `zebrad -c myconfig.toml start`; -# 2. The file `zebrad.toml` in the users's preference directory (platform-dependent); -# 3. The default config. -# -# The user's preference directory and the default path to the `zebrad` config are platform dependent, -# based on `dirs::preference_dir`, see https://docs.rs/dirs/latest/dirs/fn.preference_dir.html : -# -# | Platform | Value | Example | -# | -------- | ------------------------------------- | ---------------------------------------------- | -# | Linux | `$XDG_CONFIG_HOME` or `$HOME/.config` | `/home/alice/.config/zebrad.toml` | -# | macOS | `$HOME/Library/Preferences` | `/Users/Alice/Library/Preferences/zebrad.toml` | -# | Windows | `{FOLDERID_RoamingAppData}` | `C:\Users\Alice\AppData\Local\zebrad.toml` | - -[consensus] -checkpoint_sync = true - -[mempool] -eviction_memory_time = "1h" -tx_cost_limit = 80000000 - -[metrics] - -[mining] -debug_like_zcashd = true - -[network] -cache_dir = true -crawl_new_peer_interval = "1m 1s" -initial_mainnet_peers = [ - "dnsseed.z.cash:8233", - "dnsseed.str4d.xyz:8233", - "mainnet.seeder.zfnd.org:8233", - "mainnet.is.yolo.money:8233", -] -initial_testnet_peers = [ - "dnsseed.testnet.z.cash:18233", - "testnet.seeder.zfnd.org:18233", - "testnet.is.yolo.money:18233", -] -listen_addr = "0.0.0.0:8233" -max_connections_per_ip = 1 -network = "Mainnet" -peerset_initial_target_size = 25 - -[rpc] -cookie_dir = "cache_dir" -debug_force_finished_sync = false -enable_cookie_auth = true -parallel_cpu_threads = 0 - -[state] -cache_dir = "cache_dir" -delete_old_database = true -ephemeral = false - -[sync] -checkpoint_verify_concurrency_limit = 1000 -download_concurrency_limit = 50 -full_verify_concurrency_limit = 20 -parallel_cpu_threads = 0 - -[tracing] -buffer_limit = 128000 -force_use_color = false -use_color = true -use_journald = false - From 2ab4143874be34c66826ed4fc0b71fccf841ea2b Mon Sep 17 00:00:00 2001 From: Marek Date: Wed, 19 Feb 2025 12:29:56 +0100 Subject: [PATCH 18/20] Fix CI tests for loading of custom conf files --- .github/workflows/cd-deploy-nodes-gcp.yml | 2 +- .github/workflows/sub-ci-unit-tests-docker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd-deploy-nodes-gcp.yml b/.github/workflows/cd-deploy-nodes-gcp.yml index f7d15886293..8a3f8f3f46d 100644 --- a/.github/workflows/cd-deploy-nodes-gcp.yml +++ b/.github/workflows/cd-deploy-nodes-gcp.yml @@ -169,7 +169,7 @@ jobs: test_id: "custom-conf" docker_image: ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} test_variables: '-e ZEBRA_CONF_PATH="zebrad/tests/common/configs/v2.1.0.toml"' - grep_patterns: '-e "loaded zebrad config.*config_path.*=.*v2.1.0.toml"' + grep_patterns: '-e "extra_coinbase_data: Some(\"do you even shield?\")"' # Each time this workflow is executed, a build will be triggered to create a new image # with the corresponding tags using information from Git diff --git a/.github/workflows/sub-ci-unit-tests-docker.yml b/.github/workflows/sub-ci-unit-tests-docker.yml index ab4d9a97f1d..3cbc8675ad2 100644 --- a/.github/workflows/sub-ci-unit-tests-docker.yml +++ b/.github/workflows/sub-ci-unit-tests-docker.yml @@ -157,7 +157,7 @@ jobs: with: test_id: "custom-conf" docker_image: ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} - grep_patterns: '-e "loaded zebrad config.*config_path.*=.*v2.1.0.toml"' + grep_patterns: '-e "extra_coinbase_data: Some(\"do you even shield?\")"' test_variables: '-e ZEBRA_CONF_PATH="zebrad/tests/common/configs/v2.1.0.toml"' failure-issue: From f0637f07977f9a5f92352931b8ef83b50e297866 Mon Sep 17 00:00:00 2001 From: Marek Date: Wed, 19 Feb 2025 12:43:11 +0100 Subject: [PATCH 19/20] Use the new conf file name in CI checks --- .github/workflows/cd-deploy-nodes-gcp.yml | 2 +- .github/workflows/sub-ci-unit-tests-docker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd-deploy-nodes-gcp.yml b/.github/workflows/cd-deploy-nodes-gcp.yml index 8a3f8f3f46d..9d32112e57c 100644 --- a/.github/workflows/cd-deploy-nodes-gcp.yml +++ b/.github/workflows/cd-deploy-nodes-gcp.yml @@ -168,7 +168,7 @@ jobs: with: test_id: "custom-conf" docker_image: ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} - test_variables: '-e ZEBRA_CONF_PATH="zebrad/tests/common/configs/v2.1.0.toml"' + test_variables: '-e ZEBRA_CONF_PATH="zebrad/tests/common/configs/custom-conf.toml"' grep_patterns: '-e "extra_coinbase_data: Some(\"do you even shield?\")"' # Each time this workflow is executed, a build will be triggered to create a new image diff --git a/.github/workflows/sub-ci-unit-tests-docker.yml b/.github/workflows/sub-ci-unit-tests-docker.yml index 3cbc8675ad2..6143330e2a9 100644 --- a/.github/workflows/sub-ci-unit-tests-docker.yml +++ b/.github/workflows/sub-ci-unit-tests-docker.yml @@ -157,8 +157,8 @@ jobs: with: test_id: "custom-conf" docker_image: ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} + test_variables: '-e ZEBRA_CONF_PATH="zebrad/tests/common/configs/custom-conf.toml"' grep_patterns: '-e "extra_coinbase_data: Some(\"do you even shield?\")"' - test_variables: '-e ZEBRA_CONF_PATH="zebrad/tests/common/configs/v2.1.0.toml"' failure-issue: name: Open or update issues for main branch failures From e65c35e28fa667d963f3b4d648c68d7e9b5f0477 Mon Sep 17 00:00:00 2001 From: Marek Date: Wed, 19 Feb 2025 14:04:16 +0100 Subject: [PATCH 20/20] Use an extended regexp for custom conf CI check --- .github/workflows/cd-deploy-nodes-gcp.yml | 2 +- .github/workflows/sub-ci-unit-tests-docker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd-deploy-nodes-gcp.yml b/.github/workflows/cd-deploy-nodes-gcp.yml index 9d32112e57c..3577a70711e 100644 --- a/.github/workflows/cd-deploy-nodes-gcp.yml +++ b/.github/workflows/cd-deploy-nodes-gcp.yml @@ -169,7 +169,7 @@ jobs: test_id: "custom-conf" docker_image: ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} test_variables: '-e ZEBRA_CONF_PATH="zebrad/tests/common/configs/custom-conf.toml"' - grep_patterns: '-e "extra_coinbase_data: Some(\"do you even shield?\")"' + grep_patterns: '-e "extra_coinbase_data:\sSome\(\"do you even shield\?\"\)"' # Each time this workflow is executed, a build will be triggered to create a new image # with the corresponding tags using information from Git diff --git a/.github/workflows/sub-ci-unit-tests-docker.yml b/.github/workflows/sub-ci-unit-tests-docker.yml index 6143330e2a9..c9553e11bca 100644 --- a/.github/workflows/sub-ci-unit-tests-docker.yml +++ b/.github/workflows/sub-ci-unit-tests-docker.yml @@ -158,7 +158,7 @@ jobs: test_id: "custom-conf" docker_image: ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} test_variables: '-e ZEBRA_CONF_PATH="zebrad/tests/common/configs/custom-conf.toml"' - grep_patterns: '-e "extra_coinbase_data: Some(\"do you even shield?\")"' + grep_patterns: '-e "extra_coinbase_data:\sSome\(\"do you even shield\?\"\)"' failure-issue: name: Open or update issues for main branch failures