Skip to content

Commit b95b696

Browse files
exAspArkexAspArk
committed
Release v1.0.0-beta.1
1 parent 989c88e commit b95b696

114 files changed

Lines changed: 5791 additions & 10054 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.env.sample‎

Lines changed: 10 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,13 @@
1-
BEMIDB_PORT=54321
2-
BEMIDB_DATABASE=bemidb
3-
BEMIDB_USER=
4-
BEMIDB_PASSWORD=
5-
BEMIDB_HOST=127.0.0.1
6-
BEMIDB_INIT_SQL=./init.sql
7-
BEMIDB_LOG_LEVEL=INFO
8-
9-
# Local storage
10-
BEMIDB_STORAGE_TYPE=LOCAL
11-
BEMIDB_STORAGE_PATH=./iceberg
1+
AWS_REGION=us-west-1
2+
AWS_S3_BUCKET=[REPLACE_ME]
3+
AWS_ACCESS_KEY_ID=[REPLACE_ME]
4+
AWS_SECRET_ACCESS_KEY=[REPLACE_ME]
5+
CATALOG_DATABASE_URL=postgres://postgres:postgres@host.docker.internal:5432/catalog
126

13-
# S3 storage
14-
# BEMIDB_STORAGE_TYPE=S3
15-
# BEMIDB_STORAGE_PATH=iceberg
16-
# AWS_REGION=us-west-1
17-
# AWS_ENDPOINT=s3.amazonaws.com
18-
# AWS_S3_BUCKET=[REPLACE_ME]
19-
# AWS_ACCESS_KEY_ID=[REPLACE_ME]
20-
# AWS_SECRET_ACCESS_KEY=[REPLACE_ME]
7+
SOURCE_POSTGRES_DATABASE_URL=postgres://postgres:postgres@host.docker.internal:5432/source
218

22-
# BEMIDB_DISABLE_ANONYMOUS_ANALYTICS=true
9+
SOURCE_AMPLITUDE_API_KEY=[REPLACE_ME]
10+
SOURCE_AMPLITUDE_SECRET_KEY=[REPLACE_ME]
2311

24-
# Postgres syncing
25-
PG_DATABASE_URL=postgres://[USER]:[PASSWORD]@localhost:5432/[DATABASE]
26-
# PG_SYNC_INTERVAL=1h
27-
# PG_SCHEMA_PREFIX=mydb_
28-
# PG_INCLUDE_TABLES=public.users,public.posts
29-
# PG_EXCLUDE_TABLES=public.logs
12+
BEMIDB_LOG_LEVEL=INFO
13+
BEMIDB_DISABLE_ANONYMOUS_ANALYTICS=true

‎.github/workflows/build.yml‎

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,8 @@ jobs:
1414
- name: Checkout Code
1515
uses: actions/checkout@v4
1616

17-
- name: Set Up Go
18-
uses: actions/setup-go@v5
19-
with:
20-
go-version: '1.24.3'
17+
- name: Build test container
18+
run: docker build -t bemidb-test -f Dockerfile.test --build-arg PLATFORM=linux/amd64 .
2119

22-
- name: Install Dependencies
23-
run: go get .
24-
working-directory: ./src
25-
26-
- name: Run Tests
27-
run: go test -v ./...
28-
working-directory: ./src
20+
- name: Run tests
21+
run: docker run --rm bemidb-test

‎Dockerfile‎

Lines changed: 97 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,107 @@
1-
ARG PLATFORM
2-
ARG GOOS
3-
ARG GOARCH
1+
ARG PLATFORM=linux/arm64
42

5-
FROM --platform=$PLATFORM golang:1.24.3 AS builder
3+
FROM --platform=$PLATFORM debian:12-slim AS base
4+
5+
RUN apt-get update \
6+
&& apt-get install -y --no-install-recommends \
7+
curl ca-certificates \
8+
postgresql-client \
9+
gcc g++ libc6-dev \
10+
&& rm -rf /var/lib/apt/lists/*
11+
12+
RUN adduser --disabled-login app
613

714
WORKDIR /app
815

9-
COPY src/go.mod src/go.sum ./
10-
RUN go mod download
16+
# Install Java and Trino
1117

12-
COPY src/ .
13-
RUN CGO_ENABLED=1 GOOS=$GOOS GOARCH=$GOARCH go build -o /app/bemidb
18+
ENV JAVA_HOME /opt/jdk-24
19+
ENV PATH $JAVA_HOME/bin:$PATH
1420

15-
################################################################################
21+
RUN \
22+
ARCH=$(dpkg --print-architecture) && case "$ARCH" in \
23+
"amd64") \
24+
JDK_URL="https://download.oracle.com/java/24/archive/jdk-24.0.1_linux-x64_bin.tar.gz"; \
25+
;; \
26+
"arm64") \
27+
JDK_URL="https://download.oracle.com/java/24/archive/jdk-24.0.1_linux-aarch64_bin.tar.gz"; \
28+
;; \
29+
*) \
30+
echo "Unsupported architecture: $ARCH"; \
31+
exit 1; \
32+
;; \
33+
esac \
34+
&& curl -L "$JDK_URL" -o jdk.tar.gz \
35+
&& tar -xzf jdk.tar.gz \
36+
&& rm jdk.tar.gz \
37+
&& mkdir -p /opt \
38+
&& mv jdk-24.0.1 /opt/jdk-24 \
39+
\
40+
&& curl -L https://repo1.maven.org/maven2/io/trino/trino-server/476/trino-server-476.tar.gz -o trino-server-476.tar.gz \
41+
&& tar -xzf trino-server-476.tar.gz \
42+
&& rm trino-server-476.tar.gz \
43+
&& mv trino-server-476 /app/trino \
44+
&& mkdir -p /data/trino \
45+
&& chown -R "app:app" /app/trino /data/trino
1646

17-
FROM --platform=$PLATFORM debian:bookworm-slim
47+
# Set up Trino
1848

19-
WORKDIR /app
49+
HEALTHCHECK --interval=10s --timeout=5s --start-period=10s \
50+
CMD /app/trino/bin/health-check.sh
51+
52+
COPY --chown=app:app docker/trino/etc /etc/trino
53+
COPY --chown=app:app docker/trino/bin /app/trino/bin
54+
COPY --chown=app:app docker/trino/configure.sql /app/trino/
55+
56+
# Set up syncers and server ########################################################################
57+
58+
FROM base AS compile
59+
60+
# Install Go
61+
62+
ENV GOROOT /usr/local/go
63+
ENV GOPATH /go
64+
ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
65+
66+
RUN \
67+
ARCH=$(dpkg --print-architecture) \
68+
&& curl -L "https://go.dev/dl/go1.24.4.linux-$ARCH.tar.gz" -o go.tar.gz \
69+
&& tar -C /usr/local -xzf go.tar.gz \
70+
&& rm go.tar.gz \
71+
&& mkdir -p "$GOPATH/src" "$GOPATH/bin" \
72+
&& chmod -R 777 "$GOPATH"
73+
74+
# Compile syncers and server
75+
76+
COPY --chown=app:app src/syncer-common/go.mod src/syncer-common/go.sum /app/src/syncer-common/
77+
RUN cd /app/src/syncer-common && go mod download
78+
79+
COPY --chown=app:app src/syncer-postgres/go.mod src/syncer-postgres/go.sum /app/src/syncer-postgres/
80+
RUN cd /app/src/syncer-postgres && go mod download
81+
82+
COPY --chown=app:app src/syncer-amplitude/go.mod src/syncer-amplitude/go.sum /app/src/syncer-amplitude/
83+
RUN cd /app/src/syncer-amplitude && go mod download
84+
85+
COPY --chown=app:app src/server/go.mod src/server/go.sum /app/src/server/
86+
RUN cd /app/src/server && go mod download
87+
88+
COPY --chown=app:app src/syncer-common /app/src/syncer-common
89+
COPY --chown=app:app src/syncer-postgres /app/src/syncer-postgres
90+
COPY --chown=app:app src/syncer-amplitude /app/src/syncer-amplitude
91+
COPY --chown=app:app src/server /app/src/server
92+
93+
RUN ARCH=$(dpkg --print-architecture) \
94+
&& cd /app/src/syncer-postgres && CGO_ENABLED=1 GOOS=linux GOARCH=$ARCH go build -o /app/bin/syncer-postgres \
95+
&& cd /app/src/syncer-amplitude && CGO_ENABLED=1 GOOS=linux GOARCH=$ARCH go build -o /app/bin/syncer-amplitude \
96+
&& cd /app/src/server && CGO_ENABLED=1 GOOS=linux GOARCH=$ARCH go build -o /app/bin/server
97+
98+
# Prepare final image ##############################################################################
99+
100+
FROM base AS final
101+
102+
COPY --chown=app:app --from=compile /app/bin/syncer-postgres /app/bin/syncer-amplitude /app/bin/server /app/bin/
103+
COPY --chown=app:app docker/bin /app/bin/
20104

21-
COPY --from=builder /app/bemidb /app/bemidb
105+
USER app
22106

23-
ENTRYPOINT ["/app/bemidb"]
107+
ENTRYPOINT ["/app/bin/run.sh"]

‎Dockerfile.test‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
ARG PLATFORM=linux/arm64
2+
3+
FROM --platform=$PLATFORM debian:12-slim AS base
4+
5+
RUN apt-get update \
6+
&& apt-get install -y --no-install-recommends \
7+
curl ca-certificates \
8+
postgresql-client \
9+
gcc g++ libc6-dev \
10+
&& rm -rf /var/lib/apt/lists/*
11+
12+
RUN adduser --disabled-login app
13+
14+
WORKDIR /app
15+
16+
# Install Java and Trino
17+
18+
ENV JAVA_HOME /opt/jdk-24
19+
ENV PATH $JAVA_HOME/bin:$PATH
20+
21+
RUN \
22+
ARCH=$(dpkg --print-architecture) && case "$ARCH" in \
23+
"amd64") \
24+
JDK_URL="https://download.oracle.com/java/24/archive/jdk-24.0.1_linux-x64_bin.tar.gz"; \
25+
;; \
26+
"arm64") \
27+
JDK_URL="https://download.oracle.com/java/24/archive/jdk-24.0.1_linux-aarch64_bin.tar.gz"; \
28+
;; \
29+
*) \
30+
echo "Unsupported architecture: $ARCH"; \
31+
exit 1; \
32+
;; \
33+
esac \
34+
&& curl -L "$JDK_URL" -o jdk.tar.gz \
35+
&& tar -xzf jdk.tar.gz \
36+
&& rm jdk.tar.gz \
37+
&& mkdir -p /opt \
38+
&& mv jdk-24.0.1 /opt/jdk-24 \
39+
\
40+
&& curl -L https://repo1.maven.org/maven2/io/trino/trino-server/476/trino-server-476.tar.gz -o trino-server-476.tar.gz \
41+
&& tar -xzf trino-server-476.tar.gz \
42+
&& rm trino-server-476.tar.gz \
43+
&& mv trino-server-476 /app/trino \
44+
&& mkdir -p /data/trino \
45+
&& chown -R "app:app" /app/trino /data/trino
46+
47+
# Set up Trino
48+
49+
HEALTHCHECK --interval=10s --timeout=5s --start-period=10s \
50+
CMD /app/trino/bin/health-check.sh
51+
52+
COPY --chown=app:app docker/trino/etc /etc/trino
53+
COPY --chown=app:app docker/trino/bin /app/trino/bin
54+
COPY --chown=app:app docker/trino/configure.sql /app/trino/
55+
56+
# Set up syncers and server ########################################################################
57+
58+
FROM base AS test
59+
60+
# Install Go, MinIO, and PostgreSQL
61+
62+
ENV GOROOT /usr/local/go
63+
ENV GOPATH /go
64+
ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
65+
66+
RUN \
67+
ARCH=$(dpkg --print-architecture) \
68+
\
69+
&& curl -L "https://go.dev/dl/go1.24.4.linux-$ARCH.tar.gz" -o go.tar.gz \
70+
&& tar -C /usr/local -xzf go.tar.gz \
71+
&& rm go.tar.gz \
72+
&& mkdir -p "$GOPATH/src" "$GOPATH/bin" \
73+
&& chmod -R 777 "$GOPATH" \
74+
\
75+
&& curl -L "https://dl.min.io/server/minio/release/linux-$ARCH/minio" -o /usr/local/bin/minio \
76+
&& chmod +x /usr/local/bin/minio \
77+
&& mkdir -p /data/minio \
78+
&& chown -R "app:app" /data/minio \
79+
&& curl -L "https://dl.min.io/client/mc/release/linux-$ARCH/mc" -o /usr/local/bin/mc \
80+
&& chmod +x /usr/local/bin/mc \
81+
\
82+
&& apt-get update \
83+
&& apt-get install -y --no-install-recommends gnupg \
84+
&& curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg \
85+
&& echo "deb http://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
86+
&& apt-get update \
87+
&& apt-get install -y --no-install-recommends postgresql-16 \
88+
&& mkdir -p /var/lib/postgresql/data \
89+
&& chown -R app:app /var/lib/postgresql/data \
90+
&& chmod 700 /var/lib/postgresql/data \
91+
&& mkdir -p /var/run/postgresql \
92+
&& chown -R app:app /var/run/postgresql
93+
94+
# Prepare syncers and server
95+
96+
COPY --chown=app:app src/syncer-common/go.mod src/syncer-common/go.sum /app/src/syncer-common/
97+
RUN cd /app/src/syncer-common && go mod download
98+
99+
COPY --chown=app:app src/syncer-postgres/go.mod src/syncer-postgres/go.sum /app/src/syncer-postgres/
100+
RUN cd /app/src/syncer-postgres && go mod download
101+
102+
COPY --chown=app:app src/server/go.mod src/server/go.sum /app/src/server/
103+
RUN cd /app/src/server && go mod download
104+
105+
COPY --chown=app:app src/syncer-common /app/src/syncer-common
106+
COPY --chown=app:app src/syncer-postgres /app/src/syncer-postgres
107+
COPY --chown=app:app src/server /app/src/server
108+
109+
COPY --chown=app:app docker/bin docker/bin-test /app/bin/
110+
111+
USER app
112+
113+
ENTRYPOINT ["/app/bin/test.sh"]

‎Makefile‎

Lines changed: 33 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2,44 +2,55 @@ sh:
22
devbox --env-file .env shell
33

44
install:
5-
devbox run "cd src && go mod tidy"
5+
devbox run "cd src/syncer-common && go mod tidy && \
6+
cd ../syncer-postgres && go mod tidy && \
7+
cd ../syncer-amplitude && go mod tidy && \
8+
cd ../server && go mod tidy"
69

7-
up:
8-
devbox run --env-file .env "cd src && go run ."
10+
lint:
11+
devbox run "cd src/syncer-common && go fmt && staticcheck . && \
12+
cd ../syncer-postgres && go fmt && deadcode . && staticcheck . && \
13+
cd ../syncer-amplitude && go fmt && deadcode . && staticcheck . && \
14+
cd ../server && go fmt && deadcode . && staticcheck ."
915

10-
.PHONY: build
1116
build:
12-
rm -rf build/bemidb-* && \
13-
devbox run "./scripts/build-darwin.sh" && \
14-
./scripts/build-linux.sh
15-
16-
build-local:
17-
rm -rf build/bemidb-* && \
18-
cd src && go build -o ../build/bemidb-darwin-arm64
17+
./scripts/build-docker.sh
1918

2019
publish:
2120
./scripts/publish-docker.sh
2221

23-
sync:
24-
devbox run --env-file .env "cd src && go run . sync"
22+
build-local:
23+
docker build --build-arg PLATFORM=linux/arm64 -t bemidb:local .
24+
25+
server: build-local
26+
docker run -it --rm --env-file .env -p 54321:54321 bemidb:local server
27+
28+
syncer-postgres: build-local
29+
docker run -it --rm --env-file .env -e DESTINATION_SCHEMA_NAME=postgres bemidb:local syncer-postgres
2530

26-
test:
27-
devbox run "cd src && go test ./..."
31+
syncer-amplitude: build-local
32+
docker run -it --rm --env-file .env -e DESTINATION_SCHEMA_NAME=amplitude bemidb:local syncer-amplitude
33+
34+
bash:
35+
docker run -it --rm --env-file .env bemidb:local bash
36+
37+
build-test:
38+
docker build --build-arg PLATFORM=linux/arm64 -t bemidb:test -f Dockerfile.test .
39+
40+
test: build-test
41+
docker run -it --rm bemidb:test
2842

2943
test-function:
30-
devbox run "cd src && go test ./... -run $(FUNC)"
44+
devbox run "cd src/server && go test ./... -run $(FUNC)"
3145

3246
debug:
33-
devbox run "cd src && dlv test github.com/BemiHQ/BemiDB"
34-
35-
lint:
36-
devbox run "cd src && go fmt && deadcode . && staticcheck ."
47+
devbox run "cd src/server && dlv test github.com/BemiHQ/BemiDB"
3748

3849
console:
39-
devbox run "cd src && gore"
50+
devbox run "cd src/server && gore"
4051

4152
outdated:
42-
devbox run "cd src && go list -u -m -f '{{if and .Update (not .Indirect)}}{{.}}{{end}}' all"
53+
devbox run "cd src/server && go list -u -m -f '{{if and .Update (not .Indirect)}}{{.}}{{end}}' all"
4354

4455
.PHONY: benchmark
4556
benchmark:

0 commit comments

Comments
 (0)