feat: add dataStateAt tests (#249)

Author: agneum

.gitlab-ci.yml

@@ -222,7 +222,7 @@ build-image-swagger-latest:
     - export LATEST_TAG=$(echo ${CI_COMMIT_TAG%.*}-latest)
     - export TAGS="${DOCKER_NAME}:${LATEST_TAG}"
 
-integration-test:
+bash-test:
   <<: *only_feature
   stage: integration-test
   variables:
@@ -234,3 +234,18 @@ integration-test:
     - bash test/_cleanup.sh
   tags:
     - dle-test
+
+integration-test:
+  services:
+    - docker:dind
+  <<: *only_feature
+  stage: integration-test
+  variables:
+    # Instruct Testcontainers to use the daemon of DinD.
+    DOCKER_HOST: "tcp://docker:2375"
+    # Instruct Docker not to start over TLS.
+    DOCKER_TLS_CERTDIR: ""
+    # Improve performance with overlayfs.
+    DOCKER_DRIVER: overlay2
+  script:
+    - make test-ci-integration

.golangci.yml

@@ -99,4 +99,4 @@ issues:
   max-issues-per-linter: 0
   max-same-issues: 0
 
-  new-from-rev: 67c2feac
+  new-from-rev: 66fcab89

Makefile

@@ -51,6 +51,12 @@ build-client:
 test:
 	${GOTEST} ./...
 
+test-ci-integration:
+	GO111MODULE=on go test -race  -tags=integration ./...
+
+test-local-integration:
+	sudo GO111MODULE=on /usr/local/go/bin/go test -race  -tags=integration ./...
+
 fmt:
 	go fmt $$(go list ./... | grep -v /vendor/)
 

go.mod

@@ -4,41 +4,32 @@ go 1.15
 
 require (
 	github.com/AlekSi/pointer v1.1.0
-	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
-	github.com/Microsoft/go-winio v0.4.16 // indirect
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/araddon/dateparse v0.0.0-20210207001429-0eec95c9db7e
 	github.com/aws/aws-sdk-go v1.33.8
-	github.com/containerd/containerd v1.4.0 // indirect
 	github.com/docker/cli v0.0.0-20200721130541-80fd48bcb7e7
-	github.com/docker/distribution v2.7.1+incompatible // indirect
-	github.com/docker/docker v1.13.1
-	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/docker/go-units v0.4.0 // indirect
+	github.com/docker/docker v20.10.5+incompatible
+	github.com/docker/go-connections v0.4.0
 	github.com/dustin/go-humanize v1.0.0
 	github.com/go-ole/go-ole v1.2.4 // indirect
-	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/jackc/pgtype v1.5.0
 	github.com/jackc/pgx/v4 v4.9.0
 	github.com/lib/pq v1.8.0
 	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/opencontainers/image-spec v1.0.1
 	github.com/pkg/errors v0.9.1
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rs/xid v1.2.1
 	github.com/sergi/go-diff v1.1.0
 	github.com/sethvargo/go-password v0.2.0
 	github.com/shirou/gopsutil v2.20.9+incompatible
 	github.com/stretchr/testify v1.7.0
+	github.com/testcontainers/testcontainers-go v0.10.0
 	github.com/urfave/cli/v2 v2.1.1
 	golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392
-	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
-	google.golang.org/grpc v1.31.0 // indirect
-	gopkg.in/yaml.v2 v2.2.8
-	gotest.tools v2.2.0+incompatible // indirect
+	gopkg.in/yaml.v2 v2.4.0
 )
 
 replace github.com/docker/docker v1.13.1 => github.com/docker/engine v0.0.0-20200618181300-9dc6525e6118

go.sum

@@ -16,6 +16,7 @@ import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 
 	dblabCfg "gitlab.com/postgres-ai/database-lab/v2/pkg/config"
@@ -241,7 +242,7 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
 	}
 
 	dumpCont, err := d.dockerClient.ContainerCreate(ctx, d.buildContainerConfig(pwd), hostConfig, &network.NetworkingConfig{},
-		d.dumpContainerName(),
+		&specs.Platform{}, d.dumpContainerName(),
 	)
 	if err != nil {
 		log.Err(err)

pkg/retrieval/engine/postgres/logical/dump.go

@@ -17,6 +17,7 @@ import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 
 	dblabCfg "gitlab.com/postgres-ai/database-lab/v2/pkg/config"
@@ -152,6 +153,7 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
 		r.buildContainerConfig(pwd),
 		hostConfig,
 		&network.NetworkingConfig{},
+		&specs.Platform{},
 		r.restoreContainerName(),
 	)
 	if err != nil {

pkg/retrieval/engine/postgres/logical/restore.go

@@ -18,6 +18,7 @@ import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 
 	dblabCfg "gitlab.com/postgres-ai/database-lab/v2/pkg/config"
@@ -266,7 +267,8 @@ func (r *RestoreJob) startContainer(ctx context.Context, containerName string, c
 		return "", err
 	}
 
-	newContainer, err := r.dockerClient.ContainerCreate(ctx, containerConfig, hostConfig, &network.NetworkingConfig{}, containerName)
+	newContainer, err := r.dockerClient.ContainerCreate(ctx, containerConfig, hostConfig, &network.NetworkingConfig{},
+		&specs.Platform{}, containerName)
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to create container %s", containerName)
 	}

pkg/retrieval/engine/postgres/physical/physical.go

@@ -14,6 +14,7 @@ import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 
 	dblabCfg "gitlab.com/postgres-ai/database-lab/v2/pkg/config"
@@ -181,6 +182,7 @@ func (s *LogicalInitial) runPreprocessingQueries(ctx context.Context, dataDir st
 		s.buildContainerConfig(dataDir, patchImage, pwd),
 		hostConfig,
 		&network.NetworkingConfig{},
+		&specs.Platform{},
 		s.patchContainerName(),
 	)
 	if err != nil {

pkg/retrieval/engine/postgres/snapshot/logical.go

@@ -22,6 +22,7 @@ import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/robfig/cron/v3"
 
@@ -57,7 +58,8 @@ const (
 	promoteTargetAction  = "promote"
 
 	// WAL parsing constants.
-	walNameLen = 24
+	walNameLen  = 24
+	pgVersion10 = 10
 )
 
 var defaultRecoveryCfg = map[string]string{
@@ -533,6 +535,7 @@ func (p *PhysicalInitial) promoteInstance(ctx context.Context, clonePath string,
 		p.buildContainerConfig(clonePath, promoteImage, pwd, recoveryConfig[targetActionOption]),
 		hostConfig,
 		&network.NetworkingConfig{},
+		&specs.Platform{},
 		p.promoteContainerName(),
 	)
 
@@ -643,15 +646,17 @@ func (p *PhysicalInitial) promoteInstance(ctx context.Context, clonePath string,
 func (p *PhysicalInitial) getDSAFromWAL(ctx context.Context, pgVersion float64, containerID, cloneDir string) (string, error) {
 	log.Dbg(cloneDir)
 
-	infos, err := ioutil.ReadDir(path.Join(cloneDir, "pg_wal"))
+	walDirectory := walDir(cloneDir, pgVersion)
+
+	infos, err := ioutil.ReadDir(walDirectory)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to read the pg_wal dir")
 	}
 
 	// Walk in the reverse order.
 	for i := len(infos) - 1; i >= 0; i-- {
 		fileName := infos[i].Name()
-		walFilePath := path.Join(cloneDir, "pg_wal", fileName)
+		walFilePath := path.Join(walDirectory, fileName)
 
 		log.Dbg("Look up into file: ", walFilePath)
 
@@ -670,8 +675,18 @@ func (p *PhysicalInitial) getDSAFromWAL(ctx context.Context, pgVersion float64,
 	return "", nil
 }
 
+func walDir(cloneDir string, pgVersion float64) string {
+	dir := "pg_wal"
+
+	if pgVersion < pgVersion10 {
+		dir = "pg_xlog"
+	}
+
+	return path.Join(cloneDir, dir)
+}
+
 func (p *PhysicalInitial) parseWAL(ctx context.Context, containerID string, pgVersion float64, walFilePath string) string {
-	cmd := fmt.Sprintf("/usr/lib/postgresql/%g/bin/pg_waldump %s -r Transaction | tail -1", pgVersion, walFilePath)
+	cmd := walCommand(pgVersion, walFilePath)
 
 	output, err := tools.ExecCommandWithOutput(ctx, p.dockerClient, containerID, types.ExecConfig{
 		Cmd: []string{"sh", "-c", cmd},
@@ -691,6 +706,16 @@ func (p *PhysicalInitial) parseWAL(ctx context.Context, containerID string, pgVe
 	return parseWALLine(output)
 }
 
+func walCommand(pgVersion float64, walFilePath string) string {
+	walDumpUtil := "pg_waldump"
+
+	if pgVersion < pgVersion10 {
+		walDumpUtil = "pg_xlogdump"
+	}
+
+	return fmt.Sprintf("/usr/lib/postgresql/%g/bin/%s %s -r Transaction | tail -1", pgVersion, walDumpUtil, walFilePath)
+}
+
 func parseWALLine(line string) string {
 	const (
 		commitToken = "COMMIT"