From ccf9c8dc645caa3724363f379ddd3922ead2b93f Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Fri, 5 Aug 2022 09:19:11 -0500 Subject: [PATCH 1/3] Add apples-to-apples benchmark --- README.md | 21 ++++- benchmark/pgbouncer/Dockerfile | 27 ++++++ benchmark/pgbouncer/docker-compose.yml | 14 ++++ benchmark/pgbouncer/pgbouncer.ini | 12 +++ benchmark/pgbouncer/userlist.txt | 1 + benchmark/pgcat/Dockerfile | 14 ++++ benchmark/pgcat/docker-compose.yml | 17 ++++ benchmark/pgcat/pgcat.toml | 28 +++++++ benchmark/run_benchmark.sh | 111 +++++++++++++++++++++++++ 9 files changed, 241 insertions(+), 4 deletions(-) create mode 100644 benchmark/pgbouncer/Dockerfile create mode 100644 benchmark/pgbouncer/docker-compose.yml create mode 100644 benchmark/pgbouncer/pgbouncer.ini create mode 100644 benchmark/pgbouncer/userlist.txt create mode 100644 benchmark/pgcat/Dockerfile create mode 100644 benchmark/pgcat/docker-compose.yml create mode 100644 benchmark/pgcat/pgcat.toml create mode 100755 benchmark/run_benchmark.sh diff --git a/README.md b/README.md index 5b54a7fc..b8499f4d 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,19 @@ pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol simple && \ pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended ``` +To benchmark pgcat against pgbouncer or against directly hitting postgres you can use the benchmark script +``` +cd benchmark +./run_benchmark.sh + +# optionally you can specifiy any of the following flags +# --pgcat-only to run only pgcat benchmark +# --pgbouncer-only to run only pgbouncer benchmark +# --recompile to recompile pgcat before running the benchmark +# You can also modify benchmark/pgbouncer/pgbouncer.ini and benchmark/pgcat/pgcat.toml to change +# proxy configs used for the benchmarks +``` + See [sharding README](./tests/sharding/README.md) for sharding logic testing. | **Feature** | **Tested in CI** | **Tested manually** | **Comments** | @@ -439,7 +452,7 @@ Always good to have a base line. ``` $ pgbench -t 1000 -c 16 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 -Password: +Password: starting vacuum...end. transaction type: scaling factor: 1 @@ -453,7 +466,7 @@ tps = 139443.955722 (including connections establishing) tps = 142314.859075 (excluding connections establishing) $ pgbench -t 1000 -c 32 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 -Password: +Password: starting vacuum...end. transaction type: scaling factor: 1 @@ -467,7 +480,7 @@ tps = 150644.840891 (including connections establishing) tps = 152218.499430 (excluding connections establishing) $ pgbench -t 1000 -c 64 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 -Password: +Password: starting vacuum...end. transaction type: scaling factor: 1 @@ -481,7 +494,7 @@ tps = 152517.663404 (including connections establishing) tps = 153319.188482 (excluding connections establishing) $ pgbench -t 1000 -c 128 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 -Password: +Password: starting vacuum...end. transaction type: scaling factor: 1 diff --git a/benchmark/pgbouncer/Dockerfile b/benchmark/pgbouncer/Dockerfile new file mode 100644 index 00000000..ea3e8a34 --- /dev/null +++ b/benchmark/pgbouncer/Dockerfile @@ -0,0 +1,27 @@ +FROM debian:bullseye-slim as builder + +ARG VERSION=1.17.0 +RUN \ + cd tmp && \ + apt-get update && apt-get upgrade && apt-get install libevent-2.1-7 autoconf autoconf-doc automake bash libc-ares2 curl gcc jq libc-dev libevent-dev libtool make man libssl1.1 libssl-dev pkg-config libpq5 libpq-dev util-linux -y && \ + curl -o /tmp/pgbouncer.tar.gz -L https://pgbouncer.github.io/downloads/files/$VERSION/pgbouncer-$VERSION.tar.gz && \ + tar xvfz /tmp/pgbouncer.tar.gz && \ + mv pgbouncer-$VERSION pgbouncer && \ + cd pgbouncer && \ + ./configure --prefix=/usr && \ + make && \ + ls + +FROM debian:bullseye-slim + +RUN apt-get update && apt-get upgrade && apt-get install libevent-2.1-7 postgresql-contrib -y +COPY --from=builder /tmp/pgbouncer/pgbouncer /usr/bin/pgbouncer +COPY userlist.txt /etc/userlist.txt +RUN mkdir -p /etc/pgbouncer /var/log/pgbouncer /var/run/pgbouncer && \ + adduser --disabled-password --system pgbouncer && \ + chown pgbouncer /var/run/pgbouncer && \ + chown pgbouncer /var/log/pgbouncer + +USER pgbouncer +ENTRYPOINT ["/usr/bin/pgbouncer", "/etc/pgbouncer.ini"] + diff --git a/benchmark/pgbouncer/docker-compose.yml b/benchmark/pgbouncer/docker-compose.yml new file mode 100644 index 00000000..94434c49 --- /dev/null +++ b/benchmark/pgbouncer/docker-compose.yml @@ -0,0 +1,14 @@ +version: "3" +services: + postgres: + image: postgres:13 + environment: + POSTGRES_USER: main_user + POSTGRES_PASSWORD: main_user + POSTGRES_DB: shard0 + command: ["postgres", "-c", "max_connections=500"] + proxy: + network_mode: "service:postgres" + build: . + volumes: + - ./pgbouncer.ini:/etc/pgbouncer.ini diff --git a/benchmark/pgbouncer/pgbouncer.ini b/benchmark/pgbouncer/pgbouncer.ini new file mode 100644 index 00000000..2f7fd8c7 --- /dev/null +++ b/benchmark/pgbouncer/pgbouncer.ini @@ -0,0 +1,12 @@ +[databases] +shard0 = host=127.0.0.1 port=5432 user=main_user password=main_user dbname=shard0 max_db_connections=100 + +[pgbouncer] +pool_mode = transaction +max_client_conn = 1000 +listen_port = 6432 +listen_addr = 0.0.0.0 +logfile = /var/log/pgbouncer/log +pidfile = /var/log/pgbouncer/pid +auth_type = md5 +auth_file = /etc/userlist.txt diff --git a/benchmark/pgbouncer/userlist.txt b/benchmark/pgbouncer/userlist.txt new file mode 100644 index 00000000..5b0b32ec --- /dev/null +++ b/benchmark/pgbouncer/userlist.txt @@ -0,0 +1 @@ +"main_user" "main_user" diff --git a/benchmark/pgcat/Dockerfile b/benchmark/pgcat/Dockerfile new file mode 100644 index 00000000..52e4c261 --- /dev/null +++ b/benchmark/pgcat/Dockerfile @@ -0,0 +1,14 @@ +FROM rust:1 AS builder +WORKDIR /app +COPY Cargo.toml /app/Cargo.toml +COPY src /app/src +RUN cargo build --release + +FROM debian:bullseye-slim +COPY --from=builder /app/target/release/pgcat /usr/bin/pgcat +RUN apt-get update && apt-get install postgresql-contrib -y +COPY ./benchmark/pgcat/pgcat.toml /etc/pgcat/pgcat.toml +#COPY ./benchmark/pgcat/pgcat_old.toml /etc/pgcat/pgcat.toml +WORKDIR /etc/pgcat +ENV RUST_LOG=error +CMD ["pgcat"] diff --git a/benchmark/pgcat/docker-compose.yml b/benchmark/pgcat/docker-compose.yml new file mode 100644 index 00000000..5fac6701 --- /dev/null +++ b/benchmark/pgcat/docker-compose.yml @@ -0,0 +1,17 @@ +version: "3" +services: + postgres: + image: postgres:13 + environment: + POSTGRES_USER: main_user + POSTGRES_PASSWORD: main_user + POSTGRES_DB: shard0 + command: ["postgres", "-c", "max_connections=500"] + proxy: + network_mode: "service:postgres" + build: + context: ../../ + dockerfile: benchmark/pgcat/Dockerfile + volumes: + - ./pgcat.toml:/etc/pgcat/pgcat.toml + diff --git a/benchmark/pgcat/pgcat.toml b/benchmark/pgcat/pgcat.toml new file mode 100644 index 00000000..a2c8b4fe --- /dev/null +++ b/benchmark/pgcat/pgcat.toml @@ -0,0 +1,28 @@ +[general] +host = "0.0.0.0" +port = 6432 +connect_timeout = 5000 +healthcheck_timeout = 1000 +ban_time = 60 # seconds +autoreload = false +admin_username = "user" +admin_password = "pass" + +[pools.shard0] +pool_mode = "transaction" +default_role = "primary" +query_parser_enabled = false +primary_reads_enabled = false +sharding_function = "pg_bigint_hash" + +[pools.shard0.users.0] +username = "main_user" +password = "main_user" +pool_size = 100 + +# Shard 0 +[pools.shard0.shards.0] +servers = [ + [ "127.0.0.1", 5432, "primary" ] +] +database = "shard0" diff --git a/benchmark/run_benchmark.sh b/benchmark/run_benchmark.sh new file mode 100755 index 00000000..03c4e336 --- /dev/null +++ b/benchmark/run_benchmark.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +RECOMPILE_PGCAT=0 +PGCAT_ONLY=0 +PGBOUNCER_ONLY=0 +for ((i=1;i<=$#;i++)); +do + if [ ${!i} = "--pgcat-only" ] + then + PGCAT_ONLY=1 + elif [ ${!i} = "--pgbouncer-only" ] + then + PGBOUNCER_ONLY=1 + elif [ ${!i} = "--recompile" ] + then + RECOMPILE_PGCAT=1 + fi +done; + +wait_for_containers() { + until docker compose exec --env PGPASSWORD=main_user proxy psql -p 6432 -h 127.0.0.1 -U main_user -d shard0 -c "select 1" &> /dev/null + do + echo "Waiting for postgres server" + sleep 1 + done + sleep 2 +} + +benchmark_pgbouncer() { + echo "======================" + echo "Running Pgbouncer Test" + echo "======================" + cd pgbouncer + docker compose down -v # Remove any stored data from previous runs + docker compose up -d + wait_for_containers + + docker compose exec --env PGPASSWORD=main_user proxy pgbench -p 6432 -h 127.0.0.1 -U main_user -i shard0 + + echo "" + echo "================================================" + echo "[Pgbouncer] Running test directly against the DB" + echo "================================================" + docker compose exec --env PGPASSWORD=main_user proxy pgbench -t 1000 -c 128 -j 2 -p 5432 -h 127.0.0.1 -U main_user -S --protocol extended shard0 + + + echo "" + echo "==========================================" + echo "[Pgbouncer] Running test against the proxy" + echo "==========================================" + docker compose exec --env PGPASSWORD=main_user proxy pgbench -t 1000 -c 128 -j 2 -p 6432 -h 127.0.0.1 -U main_user -S --protocol extended shard0 + + docker compose down -v + + echo "" + echo "" + cd .. +} + +benchmark_pgcat() { + echo "==========" + echo "Pgcat Test" + echo "==========" + cd pgcat + if [ $RECOMPILE_PGCAT = 1 ]; + then + echo "Recompiling Pgcat" + docker compose build proxy --no-cache + fi + docker compose down -v # Remove any stored data from previous runs + docker compose up -d + wait_for_containers + + docker compose exec --env PGPASSWORD=main_user proxy pgbench -p 6432 -h 127.0.0.1 -U main_user -i shard0 + + echo "" + echo "============================================" + echo "[Pgcat] Running test directly against the DB" + echo "============================================" + docker compose exec --env PGPASSWORD=main_user proxy pgbench -t 1000 -c 128 -j 2 -p 5432 -h 127.0.0.1 -U main_user -S --protocol extended shard0 + + echo "" + echo "======================================" + echo "[Pgcat] Running test against the proxy" + echo "======================================" + docker compose exec --env PGPASSWORD=main_user proxy pgbench -t 1000 -c 128 -j 2 -p 6432 -h 127.0.0.1 -U main_user -S --protocol extended shard0 + + docker compose down -v + + echo "" + echo "" + + cd .. +} + + +if [ PGCAT_ONLY = 1 ] +then + benchmark_pgcat + exit 0 +fi + +if [ PGBOUNCER_ONLY = 1 ] +then + benchmark_pgbouncer + exit 0 +fi + +benchmark_pgbouncer +benchmark_pgcat + From 394f99ee817c068e8f06da6c2f70c3ebd072a93e Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Fri, 5 Aug 2022 12:58:18 -0500 Subject: [PATCH 2/3] address comments --- benchmark/pgbouncer/Dockerfile | 3 +- benchmark/pgbouncer/pgbouncer.ini | 1 + benchmark/pgcat/pgcat.toml | 2 +- benchmark/run_benchmark.sh | 50 ++++++++++++++++++------------- 4 files changed, 33 insertions(+), 23 deletions(-) diff --git a/benchmark/pgbouncer/Dockerfile b/benchmark/pgbouncer/Dockerfile index ea3e8a34..c07051cb 100644 --- a/benchmark/pgbouncer/Dockerfile +++ b/benchmark/pgbouncer/Dockerfile @@ -9,8 +9,7 @@ RUN \ mv pgbouncer-$VERSION pgbouncer && \ cd pgbouncer && \ ./configure --prefix=/usr && \ - make && \ - ls + make FROM debian:bullseye-slim diff --git a/benchmark/pgbouncer/pgbouncer.ini b/benchmark/pgbouncer/pgbouncer.ini index 2f7fd8c7..dd93fc68 100644 --- a/benchmark/pgbouncer/pgbouncer.ini +++ b/benchmark/pgbouncer/pgbouncer.ini @@ -10,3 +10,4 @@ logfile = /var/log/pgbouncer/log pidfile = /var/log/pgbouncer/pid auth_type = md5 auth_file = /etc/userlist.txt +query_wait_timeout = 120 diff --git a/benchmark/pgcat/pgcat.toml b/benchmark/pgcat/pgcat.toml index a2c8b4fe..c4fdd1b0 100644 --- a/benchmark/pgcat/pgcat.toml +++ b/benchmark/pgcat/pgcat.toml @@ -1,7 +1,7 @@ [general] host = "0.0.0.0" port = 6432 -connect_timeout = 5000 +connect_timeout = 120000 healthcheck_timeout = 1000 ban_time = 60 # seconds autoreload = false diff --git a/benchmark/run_benchmark.sh b/benchmark/run_benchmark.sh index 03c4e336..7a1f3f6c 100755 --- a/benchmark/run_benchmark.sh +++ b/benchmark/run_benchmark.sh @@ -3,6 +3,7 @@ RECOMPILE_PGCAT=0 PGCAT_ONLY=0 PGBOUNCER_ONLY=0 + for ((i=1;i<=$#;i++)); do if [ ${!i} = "--pgcat-only" ] @@ -17,7 +18,21 @@ do fi done; -wait_for_containers() { +drop_and_recreate_database() { + docker compose stop proxy &> /dev/null # Avoid errors if proxy is connected to the database + docker compose exec --env PGPASSWORD=main_user postgres psql -p 5432 -h 127.0.0.1 -U main_user -d postgres -c "DROP DATABASE shard0" &> /dev/null + docker compose exec --env PGPASSWORD=main_user postgres psql -p 5432 -h 127.0.0.1 -U main_user -d postgres -c "CREATE DATABASE shard0" &> /dev/null + docker compose start proxy &> /dev/null + wait_for_proxy +} + + +run_benchmark() { + docker compose exec --env PGPASSWORD=main_user proxy pgbench -p $1 -h 127.0.0.1 -U main_user -i shard0 + docker compose exec --env PGPASSWORD=main_user proxy pgbench -t 25000 -c 128 -j 2 -p $1 -h 127.0.0.1 -U main_user -S --protocol extended shard0 +} + +wait_for_proxy() { until docker compose exec --env PGPASSWORD=main_user proxy psql -p 6432 -h 127.0.0.1 -U main_user -d shard0 -c "select 1" &> /dev/null do echo "Waiting for postgres server" @@ -32,25 +47,22 @@ benchmark_pgbouncer() { echo "======================" cd pgbouncer docker compose down -v # Remove any stored data from previous runs - docker compose up -d - wait_for_containers - - docker compose exec --env PGPASSWORD=main_user proxy pgbench -p 6432 -h 127.0.0.1 -U main_user -i shard0 + docker compose up -d &> /dev/null + wait_for_proxy echo "" echo "================================================" echo "[Pgbouncer] Running test directly against the DB" echo "================================================" - docker compose exec --env PGPASSWORD=main_user proxy pgbench -t 1000 -c 128 -j 2 -p 5432 -h 127.0.0.1 -U main_user -S --protocol extended shard0 - + drop_and_recreate_database + run_benchmark 5432 echo "" echo "==========================================" echo "[Pgbouncer] Running test against the proxy" echo "==========================================" - docker compose exec --env PGPASSWORD=main_user proxy pgbench -t 1000 -c 128 -j 2 -p 6432 -h 127.0.0.1 -U main_user -S --protocol extended shard0 - - docker compose down -v + drop_and_recreate_database + run_benchmark 6432 echo "" echo "" @@ -68,24 +80,22 @@ benchmark_pgcat() { docker compose build proxy --no-cache fi docker compose down -v # Remove any stored data from previous runs - docker compose up -d - wait_for_containers - - docker compose exec --env PGPASSWORD=main_user proxy pgbench -p 6432 -h 127.0.0.1 -U main_user -i shard0 + docker compose up -d &> /dev/null + wait_for_proxy echo "" echo "============================================" echo "[Pgcat] Running test directly against the DB" echo "============================================" - docker compose exec --env PGPASSWORD=main_user proxy pgbench -t 1000 -c 128 -j 2 -p 5432 -h 127.0.0.1 -U main_user -S --protocol extended shard0 + drop_and_recreate_database + run_benchmark 5432 echo "" echo "======================================" echo "[Pgcat] Running test against the proxy" echo "======================================" - docker compose exec --env PGPASSWORD=main_user proxy pgbench -t 1000 -c 128 -j 2 -p 6432 -h 127.0.0.1 -U main_user -S --protocol extended shard0 - - docker compose down -v + drop_and_recreate_database + run_benchmark 6432 echo "" echo "" @@ -94,13 +104,13 @@ benchmark_pgcat() { } -if [ PGCAT_ONLY = 1 ] +if [ $PGCAT_ONLY = 1 ] then benchmark_pgcat exit 0 fi -if [ PGBOUNCER_ONLY = 1 ] +if [ $PGBOUNCER_ONLY = 1 ] then benchmark_pgbouncer exit 0 From d0dce56aac78d76c75cde54bc9bc26083232c39e Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Fri, 5 Aug 2022 13:04:55 -0500 Subject: [PATCH 3/3] trigger build