Skip to content

Commit 1ffb114

Browse files
bailinhebailinhe
authored
Create driver and migrations for spanner (psql) (#306)
* Create driver and migrations for spanner (psql) - [x] create seperate migration directories for spanner SQL files - [x] create `pgx` driver to use spanner-pg-adapter - [x] create additional config directory to configure `pgx` driver - [x] create docker compose file for building dev containers with spanener and spanner-pg-adapter Signed-off-by: Bailin He <[email protected]> * renamed enums Signed-off-by: Bailin He <[email protected]> * use spanner `.env` file Signed-off-by: Bailin He <[email protected]> * update helm chart Signed-off-by: Bailin He <[email protected]> * fix typo `db.engine` Signed-off-by: Bailin He <[email protected]> --------- Signed-off-by: Bailin He <[email protected]>
1 parent f706f45 commit 1ffb114

34 files changed

+616
-43
lines changed

‎.devcontainer/.env

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ IDENTITYAPI_TRACING_PROVIDER=jaeger
1919
IDENTITYAPI_TRACING_JAEGER_ENDPOINT=http://localhost:14268/api/traces
2020
IDENTITYAPI_CRDB_URI="postgresql://root@crdb:26257/identityapi_dev?sslmode=disable"
2121

22+
PERMISSIONSAPI_DB_ENGINE=cockroach
2223
PERMISSIONSAPI_CRDB_URI="postgresql://root@crdb:26257/permissionsapi?sslmode=disable"
23-
2424
PERMISSIONSAPI_TRACING_ENABLED=true
2525
PERMISSIONSAPI_TRACING_PROVIDER=otlpgrpc
2626
PERMISSIONSAPI_TRACING_OTLP_ENDPOINT=jaeger:4317

‎.devcontainer/devcontainer.json

+3
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
{
33
"name": "permissions-api",
44
"dockerComposeFile":"docker-compose.yml",
5+
// use the following instead if you want to use spanner and its emulator
6+
// instead of crdb
7+
// "dockerComposeFile":"spanner.docker-compose.yml",
58
"service": "app",
69
"workspaceFolder": "/workspace",
710
"shutdownAction": "stopCompose",

‎.devcontainer/spanner.docker-compose.yml

+172
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
networks:
2+
infradev:
3+
4+
services:
5+
app:
6+
build:
7+
context: .
8+
dockerfile: Dockerfile
9+
args:
10+
VARIANT: 1.23-bullseye
11+
NODE_VERSION: "none"
12+
command: sleep infinity
13+
# uncomment along with bind volume to use SSH for dev container access
14+
# ports:
15+
# - "127.0.0.1:2224:2222"
16+
env_file:
17+
- spanner.env
18+
volumes:
19+
- ./nsc:/nsc
20+
- ./nats:/nats
21+
- ..:/workspace:cached
22+
# - type: bind
23+
# source: ~/.ssh/authorized_keys
24+
# target: /home/vscode/.ssh/authorized_keys
25+
# read_only: true
26+
networks:
27+
- infradev
28+
# Use "forwardPorts" in **devcontainer.json** to forward a port locally.
29+
# (Adding the "ports" property to this file will not forward from a Codespace.)
30+
31+
# DB
32+
spanner:
33+
image: gcr.io/cloud-spanner-emulator/emulator
34+
networks:
35+
- infradev
36+
ports:
37+
- "9010:9010"
38+
- "9020:9020"
39+
40+
spanner-pg:
41+
image: gcr.io/cloud-spanner-pg-adapter/pgadapter
42+
command:
43+
- "-p test-project"
44+
- "-i test-instance"
45+
- "-r autoConfigEmulator=true"
46+
- "-e spanner:9010"
47+
- "-c \"\""
48+
- "-x"
49+
- -ddl=AutocommitExplicitTransaction
50+
ports:
51+
- "5432:5432"
52+
depends_on:
53+
- spanner
54+
networks:
55+
- infradev
56+
57+
create-databases-pg:
58+
image: postgres:13.4
59+
restart: on-failure:5
60+
command: |
61+
psql -h spanner-pg -c 'CREATE DATABASE permissionsapi;'
62+
depends_on:
63+
- spanner-pg
64+
networks:
65+
- infradev
66+
67+
create-databases-spanner:
68+
image: alpine/curl
69+
restart: on-failure:5
70+
command: |
71+
curl -X POST \
72+
http://spanner:9020/v1/projects/test-project/instances/test-instance/databases \
73+
-H 'Content-Type: application/json' \
74+
-d '{
75+
"createStatement": "CREATE DATABASE spicedb",
76+
"databaseDialect": "GOOGLE_STANDARD_SQL"
77+
}'
78+
depends_on:
79+
create-databases-pg:
80+
condition: service_completed_successfully
81+
networks:
82+
- infradev
83+
84+
create-goose-table:
85+
image: postgres:13.4
86+
depends_on:
87+
create-databases-pg:
88+
condition: service_completed_successfully
89+
restart: on-failure:5
90+
networks:
91+
- infradev
92+
command: |
93+
psql -h spanner-pg -d projects/test-project/instances/test-instance/databases/permissionsapi \
94+
-c \
95+
'CREATE TABLE IF NOT EXISTS public.goose_db_version (
96+
id BIGINT NOT NULL DEFAULT (extract(epoch from CURRENT_TIMESTAMP)*1000)::bigint,
97+
version_id BIGINT NOT NULL,
98+
is_applied BOOLEAN NOT NULL,
99+
tstamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
100+
PRIMARY KEY (id)
101+
);
102+
103+
INSERT INTO goose_db_version (version_id, is_applied) VALUES (0, true);'
104+
105+
migrate_spicedb:
106+
image: authzed/spicedb:v1.38.1
107+
command: datastore migrate head
108+
restart: on-failure:5
109+
env_file:
110+
- spanner.env
111+
depends_on:
112+
create-databases-pg:
113+
condition: service_completed_successfully
114+
networks:
115+
- infradev
116+
117+
spicedb:
118+
image: authzed/spicedb:v1.38.1
119+
command: serve
120+
restart: unless-stopped
121+
env_file:
122+
- spanner.env
123+
depends_on:
124+
migrate_spicedb:
125+
condition: service_completed_successfully
126+
ports:
127+
- 50051:50051
128+
networks:
129+
- infradev
130+
131+
nats-init:
132+
image: natsio/nats-box
133+
environment:
134+
- NSC_HOME=/nsc
135+
volumes:
136+
- ./nsc:/nsc
137+
- ./nats:/nats
138+
- ./scripts:/scripts
139+
command:
140+
- /scripts/nats_init.sh
141+
142+
nats:
143+
image: 'nats:alpine'
144+
depends_on:
145+
- nats-init
146+
command:
147+
- -c
148+
- '/etc/nats/nats-server.conf'
149+
- -D
150+
volumes:
151+
- ./nats/:/etc/nats
152+
restart: unless-stopped
153+
networks:
154+
- infradev
155+
156+
jaeger:
157+
image: jaegertracing/all-in-one:1.60.0
158+
environment:
159+
- COLLECTOR_OTLP_ENABLED=true
160+
ports:
161+
- 16688:16686
162+
networks:
163+
- infradev
164+
165+
mock-oauth2-server:
166+
image: ghcr.io/navikt/mock-oauth2-server:2.1.10
167+
networks:
168+
- infradev
169+
environment:
170+
- PORT=8081
171+
ports:
172+
- 8081:8081

‎.devcontainer/spanner.env

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# spicedb container config
2+
SPICEDB_GRPC_PRESHARED_KEY=infradev
3+
SPICEDB_DATASTORE_ENGINE=spanner
4+
SPICEDB_DATASTORE_CONN_URI='projects/test-project/instances/test-instance/databases/spicedb'
5+
SPICEDB_DATASTORE_SPANNER_EMULATOR_HOST=spanner:9010
6+
SPICEDB_LOG_LEVEL=info
7+
SPICEDB_LOG_FORMAT=console
8+
SPICEDB_OTEL_PROVIDER=jaeger
9+
SPICEDB_OTEL_INSECURE=true
10+
SPICEDB_OTEL_ENDPOINT=http://app:14268/api/traces
11+
SPICEDB_TESTING_ONLY_SCHEMA_ADDITIVE_WRITES=true
12+
13+
# zed CLI tool config
14+
ZED_ENDPOINT=spicedb:50051
15+
ZED_INSECURE=true
16+
ZED_TOKEN=infradev
17+
18+
PERMISSIONSAPI_DB_DRIVER=postgres
19+
PERMISSIONSAPI_SPANNER_URI="postgresql://spanner-pg:5432/permissionsapi?sslmode=disable"
20+
PERMISSIONSAPI_TRACING_ENABLED=true
21+
PERMISSIONSAPI_TRACING_PROVIDER=otlpgrpc
22+
PERMISSIONSAPI_TRACING_OTLP_ENDPOINT=jaeger:4317
23+
PERMISSIONSAPI_TRACING_OTLP_INSECURE=true
24+
PERMISSIONSAPI_SPICEDB_ENDPOINT=spicedb:50051
25+
PERMISSIONSAPI_SPICEDB_KEY=infradev
26+
PERMISSIONSAPI_SPICEDB_INSECURE=true
27+
28+
PERMISSIONSAPI_PUBSUB_NAME=permissionsapi
29+
PERMISSIONSAPI_PUBSUB_CREDENTIALS="/tmp/user.creds"
30+
PERMISSIONSAPI_PUBSUB_SERVER="nats://nats:4222"
31+
PERMISSIONSAPI_PUBSUB_STREAM="permissionsapi"
32+
PERMISSIONSAPI_PUBSUB_PREFIX="com.infratographer.events"
33+
34+
NATS_URL="nats://nats:4222"
35+
NATS_CREDS="/tmp/user.creds"
36+
37+
NKEYS_PATH="/workspace/.devcontainer/nsc/nkeys"
38+
NSC_HOME="/workspace/.devcontainer/nsc/nats"

‎README.md

+6
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,12 @@ To get started, you can use either [VS Code][vs-code] or the official [CLI][cli]
127127
[vs-code]: https://code.visualstudio.com/docs/devcontainers/containers
128128
[cli]: https://github.com/devcontainers/cli
129129

130+
### Spanner Emulator
131+
132+
To develop on Spanner DB emulator:
133+
134+
1. edit `.devcontainer/devcontainer.json` and use `spanner.docker-compose.yml`
135+
130136
### Manually setting up SSH agent forwarding
131137

132138
The provided dev container listens for SSH connections on port 2222 and bind mounts `~/.ssh/authorized_keys` from the host to facilitate SSH. In order to perform Git operations (i.e., committing code in the container), you will need to enable SSH agent forwarding from your machine to the dev container. While VS Code handles this automatically, for other editors you will need to set this up manually.

‎chart/permissions-api/templates/config-server.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,4 @@ metadata:
1010
service: server
1111
data:
1212
config.yaml: |
13-
{{- pick .Values.config "server" "oidc" "crdb" "spicedb" "tracing" "events" | toYaml | nindent 4 }}
13+
{{- pick .Values.config "server" "oidc" "db" "psql" "crdb" "spicedb" "tracing" "events" | toYaml | nindent 4 }}

‎chart/permissions-api/templates/config-worker.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,4 @@ metadata:
1010
service: worker
1111
data:
1212
config.yaml: |
13-
{{- pick .Values.config "server" "events" "oidc" "crdb" "spicedb" "tracing" | toYaml | nindent 4 }}
13+
{{- pick .Values.config "server" "events" "oidc" "db" "psql" "crdb" "spicedb" "tracing" | toYaml | nindent 4 }}

‎chart/permissions-api/templates/deployment-server.yaml

+12-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ spec:
4444
securityContext:
4545
{{- toYaml .Values.deployment.podSecurityContext | nindent 8 }}
4646
{{- end }}
47-
{{- if eq .Values.config.crdb.migrateHook "init" }}
47+
{{- if eq .Values.config.db.migrateHook "init" }}
4848
initContainers:
4949
- name: {{ include "common.names.name" . }}-migrate-database-init
5050
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
@@ -54,6 +54,16 @@ spec:
5454
- up
5555
- --config
5656
- /config/config.yaml
57+
{{- if eq .Values.config.db.engine "postgres"}}
58+
{{- with .Values.config.psql.uriSecretName }}
59+
env:
60+
- name: PERMISSIONSAPI_PSQL_URI
61+
valueFrom:
62+
secretKeyRef:
63+
name: {{ . }}
64+
key: uri
65+
{{- end }}
66+
{{- else }}
5767
{{- with .Values.config.crdb.uriSecretName }}
5868
env:
5969
- name: PERMISSIONSAPI_CRDB_URI
@@ -62,6 +72,7 @@ spec:
6272
name: {{ . }}
6373
key: uri
6474
{{- end }}
75+
{{- end }}
6576
{{- with .Values.deployment.resources }}
6677
resources:
6778
{{- toYaml . | nindent 12 }}

‎chart/permissions-api/templates/deployment-worker.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,13 @@ spec:
5555
env:
5656
- name: PERMISSIONSAPI_SERVER_LISTEN
5757
value: ":{{ include "permapi.listenPort" . }}"
58+
{{- with .Values.config.psql.uriSecretName }}
59+
- name: PERMISSIONSAPI_PSQL_URI
60+
valueFrom:
61+
secretKeyRef:
62+
name: {{ . }}
63+
key: uri
64+
{{- end }}
5865
{{- with .Values.config.crdb.uriSecretName }}
5966
- name: PERMISSIONSAPI_CRDB_URI
6067
valueFrom:

‎chart/permissions-api/templates/job-migrate-database.yaml

+13-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
{{- if has .Values.config.crdb.migrateHook (list "pre-sync" "manual") }}
1+
{{- if has .Values.config.db.migrateHook (list "pre-sync" "manual") }}
22
---
33
apiVersion: batch/v1
44
kind: Job
55
metadata:
6-
{{- if eq .Values.config.crdb.migrateHook "manual" }}
6+
{{- if eq .Values.config.db.migrateHook "manual" }}
77
name: {{ include "common.names.name" . }}-migrate-database
88
{{- else }}
99
generateName: migrate-database-
@@ -41,6 +41,16 @@ spec:
4141
- up
4242
- --config
4343
- /config/config.yaml
44+
{{- if eq .Values.config.db.engine "postgres"}}
45+
{{- with .Values.config.psql.uriSecretName }}
46+
env:
47+
- name: PERMISSIONSAPI_PSQL_URI
48+
valueFrom:
49+
secretKeyRef:
50+
name: {{ . }}
51+
key: uri
52+
{{- end }}
53+
{{- else }}
4454
{{- with .Values.config.crdb.uriSecretName }}
4555
env:
4656
- name: PERMISSIONSAPI_CRDB_URI
@@ -49,6 +59,7 @@ spec:
4959
name: {{ . }}
5060
key: uri
5161
{{- end }}
62+
{{- end }}
5263
{{- with .Values.deployment.resources }}
5364
resources:
5465
{{- toYaml . | nindent 12 }}

‎chart/permissions-api/values.yaml

+31-1
Original file line numberDiff line numberDiff line change
@@ -45,12 +45,42 @@ config:
4545
pskSecretName: ""
4646
# policyConfigMapName is the name of the Config Map containing the policy file configuration
4747
policyConfigMapName: ""
48-
crdb:
48+
db:
49+
# db engine to use for the permissions-api, cockroach or postgres, defaults to be cockroach
50+
engine: "cockroach"
4951
# migrateHook sets when to run database migrations. one of: pre-sync, init, manual
5052
# - pre-sync: hook runs as a job before any other changes are synced.
5153
# - init: is run as an init container to the server deployment and may run multiple times if replica count is high.
5254
# - manual: a migrate-database job will be available to triggered manually
5355
migrateHook: "init"
56+
psql:
57+
# name is the database name
58+
name: ""
59+
# host is the database host
60+
host: ""
61+
# user is the auth username to the database
62+
user: ""
63+
# password is the auth password to the database
64+
password: ""
65+
# params is the connection parameters to the database
66+
params: ""
67+
# uri is the raw uri connection string
68+
uri: ""
69+
# uriSecretName if set retrieves the `uri` from the provided secret name
70+
uriSecretName: ""
71+
# caSecretName if defined mounts database certificates from the provided secret
72+
# secrets are mounted at `caMountPath`
73+
caSecretName: ""
74+
# caMountPath is the path the caSecretName is mounted at
75+
caMountPath: /etc/ssl/crdb/
76+
connections:
77+
# max_open is the maximum number of open connections to the database
78+
max_open: 0
79+
# max_idle is the maximum number of connections in the idle connection
80+
max_idle: 0
81+
# max_lifetime is the maximum amount of time a connection may be idle
82+
max_lifetime: 0
83+
crdb:
5484
# name is the database name
5585
name: ""
5686
# host is the database host

0 commit comments

Comments
 (0)