Skip to content

Commit 1ada662

Browse files
arvi3411301arvi3411301
authored
ci: add tests for cli-migrations image (hasura#4396)
- Created new job test_and_build_cli_migrations which runs after test_and_build_cli - Build the cli-migrations and cli-migrations-v2 and save the images as tar image. - Run the test defined in each workflow v1 and v2. - Load the image that was built earlier in deploy step
1 parent 21a4dff commit 1ada662

27 files changed

+306
-34
lines changed

‎.circleci/cli-migrations-builder.dockerfile

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
FROM ubuntu:16.04
2+
3+
ARG docker_ver="17.09.0-ce"
4+
5+
RUN apt-get -y update \
6+
&& apt-get install -y curl make xz-utils git python jq \
7+
&& curl -Lo /tmp/docker-${docker_ver}.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${docker_ver}.tgz \
8+
&& tar -xz -C /tmp -f /tmp/docker-${docker_ver}.tgz \
9+
&& mv /tmp/docker/* /usr/bin \
10+
&& apt-get -y auto-remove \
11+
&& apt-get -y clean \
12+
&& rm -rf /var/lib/apt/lists/* \
13+
&& rm -rf /usr/share/doc/ \
14+
&& rm -rf /usr/share/man/ \
15+
&& rm -rf /usr/share/locale/

‎.circleci/config.yml

+61-25
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,46 @@ jobs:
312312
path: /build/_cli_output
313313
destination: cli
314314

315+
# test and build cli extension
316+
test_and_build_cli_ext:
317+
docker:
318+
- image: hasura/graphql-engine-extension-cli-builder:20200225
319+
working_directory: ~/graphql-engine
320+
steps:
321+
- attach_workspace:
322+
at: /build
323+
- *skip_job_on_ciignore
324+
- checkout
325+
- restore_cache:
326+
key:
327+
cli-ext-npm-cache-v2-{{ checksum "cli-ext/package.json" }}-{{ checksum "cli-ext/package-lock.json" }}
328+
- run:
329+
name: install dependencies
330+
working_directory: cli-ext
331+
command: make ci-deps
332+
- save_cache:
333+
key:
334+
cli-ext-npm-cache-v2-{{ checksum "cli-ext/package.json" }}-{{ checksum "cli-ext/package-lock.json" }}
335+
paths:
336+
- cli-ext/node_modules
337+
- ~/.npm
338+
- ~/.cache
339+
- run:
340+
name: build extension cli
341+
working_directory: cli-ext
342+
command: |
343+
npm install -g pkg
344+
npm run build
345+
make deploy
346+
make ci-copy-assets
347+
- store_artifacts:
348+
path: /build/_cli_ext_output
349+
destination: cli_ext
350+
- persist_to_workspace:
351+
root: /build
352+
paths:
353+
- _cli_ext_output
354+
315355
# test and build cli
316356
test_and_build_cli:
317357
docker:
@@ -356,44 +396,35 @@ jobs:
356396
paths:
357397
- _cli_output
358398

359-
test_and_build_cli_ext:
399+
test_and_build_cli_migrations:
360400
docker:
361-
- image: hasura/graphql-engine-extension-cli-builder:20200225
401+
- image: hasura/graphql-engine-cli-migrations-builder:20200415
362402
working_directory: ~/graphql-engine
363403
steps:
364404
- attach_workspace:
365405
at: /build
366406
- *skip_job_on_ciignore
367407
- checkout
368-
- restore_cache:
369-
key:
370-
cli-ext-npm-cache-v2-{{ checksum "cli-ext/package.json" }}-{{ checksum "cli-ext/package-lock.json" }}
408+
- *setup_remote_docker
371409
- run:
372-
name: install dependencies
373-
working_directory: cli-ext
374-
command: make ci-deps
375-
- save_cache:
376-
key:
377-
cli-ext-npm-cache-v2-{{ checksum "cli-ext/package.json" }}-{{ checksum "cli-ext/package-lock.json" }}
378-
paths:
379-
- cli-ext/node_modules
380-
- ~/.npm
381-
- ~/.cache
410+
name: Install Docker Compose
411+
command: |
412+
set -x
413+
curl -L https://github.com/docker/compose/releases/download/1.25.3/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose
414+
chmod +x /usr/local/bin/docker-compose
382415
- run:
383-
name: build extension cli
384-
working_directory: cli-ext
416+
name: Build cli-migrations image
417+
working_directory: scripts/cli-migrations
385418
command: |
386-
npm install -g pkg
387-
npm run build
388-
make deploy
389-
make ci-copy-assets
419+
mkdir -p /build/_cli_migrations_output
420+
make all
390421
- store_artifacts:
391-
path: /build/_cli_ext_output
392-
destination: cli_ext
422+
path: /build/_cli_migrations_output
423+
destination: cli-migrations
393424
- persist_to_workspace:
394425
root: /build
395426
paths:
396-
- _cli_ext_output
427+
- _cli_migrations_output
397428

398429
# build console assets
399430
build_console:
@@ -531,7 +562,7 @@ jobs:
531562

532563
workflows:
533564
version: 2
534-
workflow_v20200120:
565+
workflow_v20200414:
535566
jobs:
536567
- check_build_worthiness: *filter_only_vtags
537568
- test_and_build_cli_ext:
@@ -599,6 +630,10 @@ workflows:
599630
requires:
600631
- test_and_build_cli_ext
601632
- build_server
633+
- test_and_build_cli_migrations:
634+
<<: *filter_only_vtags
635+
requires:
636+
- test_and_build_cli
602637
- test_console:
603638
<<: *filter_only_vtags
604639
requires:
@@ -610,4 +645,5 @@ workflows:
610645
requires:
611646
- build_image
612647
- test_console
648+
- test_and_build_cli_migrations
613649
- all_server_tests_pass

‎.circleci/deploy.sh

+4-7
Original file line numberDiff line numberDiff line change
@@ -136,19 +136,16 @@ deploy_cli_ext() {
136136
# build and push container for auto-migrations
137137
build_and_push_cli_migrations_image_v1() {
138138
IMAGE_TAG="hasura/graphql-engine:${CIRCLE_TAG}.cli-migrations"
139-
cd "$ROOT/scripts/cli-migrations/v1"
140-
cp /build/_cli_output/binaries/cli-hasura-linux-amd64 .
141-
docker build -t "$IMAGE_TAG" .
139+
docker load -i /build/_cli_migrations_output/v1.tar
140+
docker tag cli-migrations "$IMAGE_TAG"
142141
docker push "$IMAGE_TAG"
143142
}
144143

145144
# build and push container for auto-migrations-v2
146145
build_and_push_cli_migrations_image_v2() {
147146
IMAGE_TAG="hasura/graphql-engine:${CIRCLE_TAG}.cli-migrations-v2"
148-
cd "$ROOT/scripts/cli-migrations/v2"
149-
cp /build/_cli_output/binaries/cli-hasura-linux-amd64 .
150-
cp /build/_cli_ext_output/manifest-dev.yaml manifest.yaml
151-
docker build -t "$IMAGE_TAG" .
147+
docker load -i /build/_cli_migrations_output/v2.tar
148+
docker tag cli-migrations-v2 "$IMAGE_TAG"
152149
docker push "$IMAGE_TAG"
153150
}
154151

‎cli/build/builder.dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:1.10
1+
FROM golang:1.13
22

33
# setup the working directory
44
WORKDIR /go/src/github.com/hasura/graphql-engine/cli

‎scripts/cli-migrations/Makefile

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
PWD := $(shell pwd)
2+
PARENT_DIR := $(shell dirname $(PWD))
3+
VERSION ?= $(shell ../get-version.sh)
4+
BUILD_DIR ?= /build
5+
BINARY ?= $(BUILD_DIR)/_cli_output/binaries/cli-hasura-linux-amd64
6+
IMAGE_TAG ?= cli-migrations
7+
BUILD_OUTPUT ?= $(BUILD_DIR)/_cli_migrations_output
8+
CLI_EXT_MANIFEST_FILE ?= $(BUILD_DIR)/_cli_ext_output/manifest-dev.yaml
9+
10+
.PHONY: build-cli-migrations-v1
11+
.ONESHELL:
12+
build-cli-migrations-v1:
13+
cd v1
14+
cp ${BINARY} .
15+
docker build -t ${IMAGE_TAG} .
16+
docker save -o '$(BUILD_OUTPUT)/v1.tar' '$(IMAGE_TAG)'
17+
18+
.PHONY: test-cli-migrations-v1
19+
.ONESHELL:
20+
test-cli-migrations-v1:
21+
cd v1/test
22+
./test.sh
23+
24+
.PHONY: build-cli-migrations-v2
25+
.ONESHELL:
26+
build-cli-migrations-v2:
27+
cd v2
28+
cp ${BINARY} .
29+
cp ${CLI_EXT_MANIFEST_FILE} manifest.yaml
30+
docker build -t '${IMAGE_TAG}-v2' .
31+
docker save -o '$(BUILD_OUTPUT)/v2.tar' '$(IMAGE_TAG)-v2'
32+
33+
.PHONY: test-cli-migrations-v2
34+
.ONESHELL:
35+
test-cli-migrations-v2:
36+
cd v2/test
37+
./test.sh
38+
39+
.PHONY: all
40+
all: build-cli-migrations-v1 build-cli-migrations-v2 test-cli-migrations-v1 test-cli-migrations-v2

‎scripts/cli-migrations/v1/test/docker-compose.yaml

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
version: '3.6'
2+
services:
3+
postgres:
4+
image: postgres:12
5+
restart: always
6+
environment:
7+
POSTGRES_PASSWORD: postgrespassword
8+
graphql-engine:
9+
container_name: graphql-engine
10+
image: cli-migrations
11+
ports:
12+
- "8080:8080"
13+
depends_on:
14+
- "postgres"
15+
restart: always
16+
environment:
17+
HASURA_GRAPHQL_DATABASE_URL: postgres://postgres:postgrespassword@postgres:5432/postgres
18+
HASURA_GRAPHQL_ENABLE_CONSOLE: "true" # set to "false" to disable console
19+
HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log

‎scripts/cli-migrations/v1/test/migrations/1586822388261_create_table_public_test/down.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
- args:
2+
cascade: false
3+
read_only: false
4+
sql: DROP TABLE "public"."test";
5+
type: run_sql

‎scripts/cli-migrations/v1/test/migrations/1586822388261_create_table_public_test/up.yaml

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
- args:
2+
cascade: false
3+
read_only: false
4+
sql: CREATE TABLE "public"."test"("id" serial NOT NULL, PRIMARY KEY ("id") );
5+
type: run_sql
6+
- args:
7+
name: test
8+
schema: public
9+
type: add_existing_table_or_view

‎scripts/cli-migrations/v1/test/migrations/metadata.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
version: 2
2+
tables:
3+
- table:
4+
schema: public
5+
name: test

‎scripts/cli-migrations/v1/test/test.sh

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/usr/bin/env bash
2+
3+
set -evo pipefail
4+
IFS=$'\n\t'
5+
ROOT="$(readlink -f ${BASH_SOURCE[0]%/*}/../../)"
6+
7+
wait_for_server() {
8+
echo "waiting for server"
9+
for _ in $(seq 1 60);
10+
do
11+
docker run --network container:graphql-engine appropriate/curl http://127.0.0.1:8080/v1/version && return
12+
echo -n .
13+
sleep 1
14+
done
15+
echo "Failed waiting for server" && exit 1
16+
}
17+
18+
# start postgres
19+
docker-compose up --no-start graphql-engine
20+
# copy migrations directory to /hasura-migrations
21+
docker cp migrations/. graphql-engine:/hasura-migrations
22+
# start graphql-engine
23+
docker-compose up -d --no-recreate graphql-engine
24+
wait_for_server
25+
# export metadata and run diff with validation/metadata.json
26+
docker run --network container:graphql-engine appropriate/curl -s -f -d'{"type" : "export_metadata", "args" : {} }' localhost:8080/v1/query | jq -j '.' | diff validation/metadata.json -
27+
# get list of migrations applied from graphql-engine server
28+
docker run --network container:graphql-engine appropriate/curl -s -f -d'{"type" : "run_sql", "args" : {"sql": "select * from hdb_catalog.schema_migrations"} }' localhost:8080/v1/query | jq -j '.' | diff validation/schema_migrations.json -
29+
# delete postgres and graphql-engine
30+
docker-compose down -v

‎scripts/cli-migrations/v1/test/validation/metadata.json

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"version": 2,
3+
"tables": [
4+
{
5+
"table": {
6+
"schema": "public",
7+
"name": "test"
8+
}
9+
}
10+
]
11+
}

‎scripts/cli-migrations/v1/test/validation/schema_migrations.json

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"result_type": "TuplesOk",
3+
"result": [
4+
[
5+
"version",
6+
"dirty"
7+
],
8+
[
9+
"1586822388261",
10+
"f"
11+
]
12+
]
13+
}

‎scripts/cli-migrations/v2/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ RUN apt-get update && apt-get download libstdc++6
44

55
FROM hasura/graphql-engine:v1.1.1
66

7-
# install libstdc++6
7+
# install libstdc++6 from .deb file
88
COPY --from=packager /tmp/libstdc++6* .
99
RUN busybox dpkg-deb -x libstdc++6*.deb / \
1010
&& rm libstdc++6*.deb

‎scripts/cli-migrations/v2/test/docker-compose.yaml

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
version: '3.6'
2+
services:
3+
postgres:
4+
image: postgres:12
5+
restart: always
6+
environment:
7+
POSTGRES_PASSWORD: postgrespassword
8+
graphql-engine:
9+
container_name: graphql-engine
10+
image: cli-migrations-v2
11+
ports:
12+
- "8080:8080"
13+
depends_on:
14+
- "postgres"
15+
restart: always
16+
environment:
17+
HASURA_GRAPHQL_DATABASE_URL: postgres://postgres:postgrespassword@postgres:5432/postgres
18+
HASURA_GRAPHQL_ENABLE_CONSOLE: "true" # set to "false" to disable console
19+
HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log

‎scripts/cli-migrations/v2/test/metadata/actions.graphql

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+

‎scripts/cli-migrations/v2/test/metadata/actions.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
actions: []
2+
custom_types:
3+
enums: []
4+
input_objects: []
5+
objects: []
6+
scalars: []

‎scripts/cli-migrations/v2/test/metadata/allow_list.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[]

‎scripts/cli-migrations/v2/test/metadata/functions.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[]

‎scripts/cli-migrations/v2/test/metadata/query_collections.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[]

‎scripts/cli-migrations/v2/test/metadata/remote_schemas.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[]

‎scripts/cli-migrations/v2/test/metadata/tables.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
- table:
2+
schema: public
3+
name: test

‎scripts/cli-migrations/v2/test/metadata/version.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
version: 2

‎scripts/cli-migrations/v2/test/migrations/1586823136625_create_table_public_test/down.sql

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
DROP TABLE "public"."test";

‎scripts/cli-migrations/v2/test/migrations/1586823136625_create_table_public_test/up.sql

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
CREATE TABLE "public"."test"("id" serial NOT NULL, PRIMARY KEY ("id") );

0 commit comments

Comments
 (0)