Merge branch 'master' into pg-db-dump-truncate-fix

Author: dhui

.github/workflows/ci.yaml

@@ -9,10 +9,10 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
-          go-version: 1.19
-      - uses: actions/checkout@v3
+          go-version: "1.21.x"
+      - uses: actions/checkout@v4
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
@@ -22,11 +22,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: ["1.18.x", "1.19.x"]
+        go: ["1.20.x", "1.21.x"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go }}
 
@@ -60,27 +60,27 @@ jobs:
     # 3. When the workflow is triggered by a tag with `v` prefix
     if: ${{ success() && github.repository == 'golang-migrate/migrate' && startsWith(github.ref, 'refs/tags/v') }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: 2.7
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
-          go-version: "1.19.x"
+          go-version: "1.21.x"
 
-      - uses: docker/setup-qemu-action@v1
-      - uses: docker/setup-buildx-action@v1
-      - uses: docker/login-action@v1
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
         with:
           username: golangmigrate
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - run: echo "SOURCE=$(make echo-source)" >> $GITHUB_ENV
       - run: echo "DATABASE=$(make echo-database)" >> $GITHUB_ENV
 
-      - uses: goreleaser/goreleaser-action@v2
+      - uses: goreleaser/goreleaser-action@v5
         with:
           version: latest
           args: release --rm-dist
@@ -94,9 +94,15 @@ jobs:
       - run: package_cloud push golang-migrate/migrate/ubuntu/focal dist/migrate.linux-amd64.deb
         env:
           PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
+      - run: package_cloud push golang-migrate/migrate/ubuntu/jammy dist/migrate.linux-amd64.deb
+        env:
+          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
       - run: package_cloud push golang-migrate/migrate/debian/buster dist/migrate.linux-amd64.deb
         env:
           PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
       - run: package_cloud push golang-migrate/migrate/debian/bullseye dist/migrate.linux-amd64.deb
         env:
           PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
+      - run: package_cloud push golang-migrate/migrate/debian/bookworm dist/migrate.linux-amd64.deb
+        env:
+          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}

.goreleaser.yml

@@ -87,6 +87,7 @@ release:
   prerelease: auto
 source:
   enabled: true
+  rlcp: true
   format: zip
 changelog:
   skip: false

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19-alpine3.16 AS builder
+FROM golang:1.21-alpine3.19 AS builder
 ARG VERSION
 
 RUN apk add --no-cache git gcc musl-dev make
@@ -15,7 +15,7 @@ COPY . ./
 
 RUN make build-docker
 
-FROM alpine:3.16
+FROM alpine:3.19
 
 RUN apk add --no-cache ca-certificates
 

Dockerfile.github-actions

@@ -1,4 +1,4 @@
-FROM alpine:3.16
+FROM alpine:3.19
 
 RUN apk add --no-cache ca-certificates
 

FAQ.md

@@ -50,7 +50,7 @@
   and whenever we want, not just once at the beginning of all tests.
 
 #### Can I maintain my driver in my own repository?
-  Yes, technically thats possible. We want to encourage you to contribute your driver to this respository though.
+  Yes, technically thats possible. We want to encourage you to contribute your driver to this repository though.
   The driver's functionality is dictated by migrate's interfaces. That means there should really
   just be one driver for a database/ source. We want to prevent a future where several drivers doing the exact same thing,
   just implemented a bit differently, co-exist somewhere on GitHub. If users have to do research first to find the

GETTING_STARTED.md

@@ -28,7 +28,7 @@ migrate -database YOUR_DATABASE_URL -path PATH_TO_YOUR_MIGRATIONS up
 
 Just add the code to your app and you're ready to go!
 
-Before commiting your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways.
+Before committing your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways.
 (e.g. if you created a table in a migration but reverse migration did not delete it, you will encounter an error when running the forward migration again)
 It's also worth checking your migrations in a separate, containerized environment. You can find some tools at the [end of this document](#further-reading).
 

Makefile

@@ -1,5 +1,5 @@
 SOURCE ?= file go_bindata github github_ee bitbucket aws_s3 google_cloud_storage godoc_vfs gitlab
-DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx
+DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx pgx5 rqlite
 DATABASE_TEST ?= $(DATABASE) sqlite sqlite3 sqlcipher
 VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-)
 TEST_FLAGS ?=

README.md

@@ -1,9 +1,9 @@
-[![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/golang-migrate/migrate/CI/master)](https://github.com/golang-migrate/migrate/actions/workflows/ci.yaml?query=branch%3Amaster)
+[![GitHub Workflow Status (branch)](https://img.shields.io/github/actions/workflow/status/golang-migrate/migrate/ci.yaml?branch=master)](https://github.com/golang-migrate/migrate/actions/workflows/ci.yaml?query=branch%3Amaster)
 [![GoDoc](https://pkg.go.dev/badge/github.com/golang-migrate/migrate)](https://pkg.go.dev/github.com/golang-migrate/migrate/v4)
 [![Coverage Status](https://img.shields.io/coveralls/github/golang-migrate/migrate/master.svg)](https://coveralls.io/github/golang-migrate/migrate?branch=master)
 [![packagecloud.io](https://img.shields.io/badge/deb-packagecloud.io-844fec.svg)](https://packagecloud.io/golang-migrate/migrate?filter=debs)
 [![Docker Pulls](https://img.shields.io/docker/pulls/migrate/migrate.svg)](https://hub.docker.com/r/migrate/migrate/)
-![Supported Go Versions](https://img.shields.io/badge/Go-1.18%2C%201.19-lightgrey.svg)
+![Supported Go Versions](https://img.shields.io/badge/Go-1.19%2C%201.20-lightgrey.svg)
 [![GitHub Release](https://img.shields.io/github/release/golang-migrate/migrate.svg)](https://github.com/golang-migrate/migrate/releases)
 [![Go Report Card](https://goreportcard.com/badge/github.com/golang-migrate/migrate/v4)](https://goreportcard.com/report/github.com/golang-migrate/migrate/v4)
 
@@ -24,7 +24,8 @@ Forked from [mattes/migrate](https://github.com/mattes/migrate)
 Database drivers run migrations. [Add a new database?](database/driver.go)
 
 * [PostgreSQL](database/postgres)
-* [PGX](database/pgx)
+* [PGX v4](database/pgx)
+* [PGX v5](database/pgx/v5)
 * [Redshift](database/redshift)
 * [Ql](database/ql)
 * [Cassandra](database/cassandra)
@@ -42,6 +43,7 @@ Database drivers run migrations. [Add a new database?](database/driver.go)
 * [ClickHouse](database/clickhouse)
 * [Firebird](database/firebird)
 * [MS SQL Server](database/sqlserver)
+* [RQLite](database/rqlite)
 
 ### Database URLs
 
@@ -169,6 +171,11 @@ Each migration has an up and down migration. [Why?](FAQ.md#why-two-separate-file
 
 [Best practices: How to write migrations.](MIGRATIONS.md)
 
+## Coming from another db migration tool?
+
+Check out [migradaptor](https://github.com/musinit/migradaptor/).
+*Note: migradaptor is not affliated or supported by this project*
+
 ## Versions
 
 Version | Supported? | Import | Notes

cmd/migrate/README.md

@@ -7,7 +7,7 @@
 [Release Downloads](https://github.com/golang-migrate/migrate/releases)
 
 ```bash
-$ curl -L https://github.com/golang-migrate/migrate/releases/download/$version/migrate.$platform-amd64.tar.gz | tar xvz
+$ curl -L https://github.com/golang-migrate/migrate/releases/download/$version/migrate.$os-$arch.tar.gz | tar xvz
 ```
 
 ### MacOS

database/parse_test.go

@@ -8,13 +8,15 @@ import (
 )
 
 const reservedChars = "!#$%&'()*+,/:;=?@[]"
+const reservedCharTestNamePrefix = "reserved char "
 
 const baseUsername = "username"
 
+const scheme = "database://"
+
 // TestUserUnencodedReservedURLChars documents the behavior of using unencoded reserved characters in usernames with
 // net/url Parse()
 func TestUserUnencodedReservedURLChars(t *testing.T) {
-	scheme := "database://"
 	urlSuffix := "password@localhost:12345/myDB?someParam=true"
 	urlSuffixAndSep := ":" + urlSuffix
 
@@ -64,7 +66,7 @@ func TestUserUnencodedReservedURLChars(t *testing.T) {
 	testedChars := make([]string, 0, len(reservedChars))
 	for _, tc := range testcases {
 		testedChars = append(testedChars, tc.char)
-		t.Run("reserved char "+tc.char, func(t *testing.T) {
+		t.Run(reservedCharTestNamePrefix+tc.char, func(t *testing.T) {
 			s := scheme + baseUsername + tc.char + urlSuffixAndSep
 			u, err := url.Parse(s)
 			if err == nil {
@@ -98,13 +100,12 @@ func TestUserUnencodedReservedURLChars(t *testing.T) {
 }
 
 func TestUserEncodedReservedURLChars(t *testing.T) {
-	scheme := "database://"
 	urlSuffix := "password@localhost:12345/myDB?someParam=true"
 	urlSuffixAndSep := ":" + urlSuffix
 
 	for _, c := range reservedChars {
 		c := string(c)
-		t.Run("reserved char "+c, func(t *testing.T) {
+		t.Run(reservedCharTestNamePrefix+c, func(t *testing.T) {
 			encodedChar := "%" + hex.EncodeToString([]byte(c))
 			s := scheme + baseUsername + encodedChar + urlSuffixAndSep
 			expectedUsername := baseUsername + c
@@ -126,7 +127,7 @@ func TestUserEncodedReservedURLChars(t *testing.T) {
 // with net/url Parse()
 func TestPasswordUnencodedReservedURLChars(t *testing.T) {
 	username := baseUsername
-	schemeAndUsernameAndSep := "database://" + username + ":"
+	schemeAndUsernameAndSep := scheme + username + ":"
 	basePassword := "password"
 	urlSuffixAndSep := "@localhost:12345/myDB?someParam=true"
 
@@ -174,7 +175,7 @@ func TestPasswordUnencodedReservedURLChars(t *testing.T) {
 	testedChars := make([]string, 0, len(reservedChars))
 	for _, tc := range testcases {
 		testedChars = append(testedChars, tc.char)
-		t.Run("reserved char "+tc.char, func(t *testing.T) {
+		t.Run(reservedCharTestNamePrefix+tc.char, func(t *testing.T) {
 			s := schemeAndUsernameAndSep + basePassword + tc.char + urlSuffixAndSep
 			u, err := url.Parse(s)
 			if err == nil {
@@ -213,13 +214,13 @@ func TestPasswordUnencodedReservedURLChars(t *testing.T) {
 
 func TestPasswordEncodedReservedURLChars(t *testing.T) {
 	username := baseUsername
-	schemeAndUsernameAndSep := "database://" + username + ":"
+	schemeAndUsernameAndSep := scheme + username + ":"
 	basePassword := "password"
 	urlSuffixAndSep := "@localhost:12345/myDB?someParam=true"
 
 	for _, c := range reservedChars {
 		c := string(c)
-		t.Run("reserved char "+c, func(t *testing.T) {
+		t.Run(reservedCharTestNamePrefix+c, func(t *testing.T) {
 			encodedChar := "%" + hex.EncodeToString([]byte(c))
 			s := schemeAndUsernameAndSep + basePassword + encodedChar + urlSuffixAndSep
 			expectedPassword := basePassword + c

database/pgx/README.md

@@ -1,5 +1,7 @@
 # pgx
 
+This package is for [pgx/v4](https://pkg.go.dev/github.com/jackc/pgx/v4). A backend for the newer [pgx/v5](https://pkg.go.dev/github.com/jackc/pgx/v5) is [also available](v5).
+
 `pgx://user:password@host:port/dbname?query`
 
 | URL Query  | WithInstance Config | Description |

database/pgx/pgx.go

@@ -28,6 +28,7 @@ import (
 func init() {
 	db := Postgres{}
 	database.Register("pgx", &db)
+	database.Register("pgx4", &db)
 }
 
 var (
@@ -150,7 +151,7 @@ func (p *Postgres) Open(url string) (database.Driver, error) {
 	// i.e. pgx://user:password@host:port/db => postgres://user:password@host:port/db
 	purl.Scheme = "postgres"
 
-	db, err := sql.Open("pgx", migrate.FilterCustomQuery(purl).String())
+	db, err := sql.Open("pgx/v4", migrate.FilterCustomQuery(purl).String())
 	if err != nil {
 		return nil, err
 	}

database/pgx/pgx_test.go

@@ -8,19 +8,17 @@ import (
 	sqldriver "database/sql/driver"
 	"errors"
 	"fmt"
-	"log"
-
 	"io"
+	"log"
 	"strconv"
 	"strings"
 	"sync"
 	"testing"
 
-	"github.com/golang-migrate/migrate/v4"
-
 	"github.com/dhui/dktest"
-
+	"github.com/golang-migrate/migrate/v4"
 	"github.com/golang-migrate/migrate/v4/database"
+
 	dt "github.com/golang-migrate/migrate/v4/database/testing"
 	"github.com/golang-migrate/migrate/v4/dktesting"
 	_ "github.com/golang-migrate/migrate/v4/source/file"
@@ -41,6 +39,9 @@ var (
 		{ImageName: "postgres:10", Options: opts},
 		{ImageName: "postgres:11", Options: opts},
 		{ImageName: "postgres:12", Options: opts},
+		{ImageName: "postgres:13", Options: opts},
+		{ImageName: "postgres:14", Options: opts},
+		{ImageName: "postgres:15", Options: opts},
 	}
 )
 

database/pgx/v5/README.md

@@ -0,0 +1,41 @@
+# pgx
+
+This package is for [pgx/v5](https://pkg.go.dev/github.com/jackc/pgx/v5). A backend for the older [pgx/v4](https://pkg.go.dev/github.com/jackc/pgx/v4). is [also available](..).
+
+`pgx5://user:password@host:port/dbname?query`
+
+| URL Query  | WithInstance Config | Description |
+|------------|---------------------|-------------|
+| `x-migrations-table` | `MigrationsTable` | Name of the migrations table |
+| `x-migrations-table-quoted` | `MigrationsTableQuoted` | By default, migrate quotes the migration table for SQL injection safety reasons. This option disable quoting and naively checks that you have quoted the migration table name. e.g. `"my_schema"."schema_migrations"` |
+| `x-statement-timeout` | `StatementTimeout` | Abort any statement that takes more than the specified number of milliseconds |
+| `x-multi-statement` | `MultiStatementEnabled` | Enable multi-statement execution (default: false) |
+| `x-multi-statement-max-size` | `MultiStatementMaxSize` | Maximum size of single statement in bytes (default: 10MB) |
+| `dbname` | `DatabaseName` | The name of the database to connect to |
+| `search_path` | | This variable specifies the order in which schemas are searched when an object is referenced by a simple name with no schema specified. |
+| `user` | | The user to sign in as |
+| `password` | | The user's password |
+| `host` | | The host to connect to. Values that start with / are for unix domain sockets. (default is localhost) |
+| `port` | | The port to bind to. (default is 5432) |
+| `fallback_application_name` | | An application_name to fall back to if one isn't provided. |
+| `connect_timeout` | | Maximum wait for connection, in seconds. Zero or not specified means wait indefinitely. |
+| `sslcert` | | Cert file location. The file must contain PEM encoded data. |
+| `sslkey` | | Key file location. The file must contain PEM encoded data. |
+| `sslrootcert` | | The location of the root certificate file. The file must contain PEM encoded data. |
+| `sslmode` | | Whether or not to use SSL (disable\|require\|verify-ca\|verify-full) |
+
+
+## Upgrading from v1
+
+1. Write down the current migration version from schema_migrations
+1. `DROP TABLE schema_migrations`
+2. Wrap your existing migrations in transactions ([BEGIN/COMMIT](https://www.postgresql.org/docs/current/static/transaction-iso.html)) if you use multiple statements within one migration.
+3. Download and install the latest migrate version.
+4. Force the current migration version with `migrate force <current_version>`.
+
+## Multi-statement mode
+
+In PostgreSQL running multiple SQL statements in one `Exec` executes them inside a transaction. Sometimes this
+behavior is not desirable because some statements can be only run outside of transaction (e.g.
+`CREATE INDEX CONCURRENTLY`). If you want to use `CREATE INDEX CONCURRENTLY` without activating multi-statement mode
+you have to put such statements in a separate migration files.