Add Bitbucket Cloud Support as source (#450)

* Added bitbucket cloud support as source for migration

* added test

* address PR Comments

* Add entry in README.md; update go.mod and go.sum

Co-authored-by: abhigupta91 <[email protected]>

Author: 2 people

Makefile

@@ -1,4 +1,4 @@
-SOURCE ?= file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab
+SOURCE ?= file go_bindata github github_ee bitbucket aws_s3 google_cloud_storage godoc_vfs gitlab
 DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver firebird neo4j
 DATABASE_TEST ?= $(DATABASE) sqlite sqlcipher
 VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-)

README.md

@@ -69,6 +69,7 @@ Source drivers read migrations from local or remote sources. [Add a new source?]
 * [Go-Bindata](source/go_bindata) - read from embedded binary data ([jteeuwen/go-bindata](https://github.com/jteeuwen/go-bindata))
 * [GitHub](source/github) - read from remote GitHub repositories
 * [GitHub Enterprise](source/github_ee) - read from remote GitHub Enterprise repositories
+* [Bitbucket](source/bitbucket) - read from remote Bitbucket repositories
 * [Gitlab](source/gitlab) - read from remote Gitlab repositories
 * [AWS S3](source/aws_s3) - read from Amazon Web Services S3
 * [Google Cloud Storage](source/google_cloud_storage) - read from Google Cloud Platform Storage

go.mod

@@ -29,6 +29,7 @@ require (
 	github.com/hashicorp/go-multierror v1.1.0
 	github.com/jackc/pgconn v1.3.2 // indirect
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
+	github.com/ktrysmt/go-bitbucket v0.6.4
 	github.com/lib/pq v1.3.0
 	github.com/markbates/pkger v0.15.1
 	github.com/mattn/go-sqlite3 v1.10.0

go.sum

@@ -150,6 +150,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw=
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -262,6 +263,9 @@ github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhB
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
+github.com/k0kubun/pp v2.3.0+incompatible h1:EKhKbi34VQDWJtq+zpsKSEhkHHs9w2P8Izbq8IhLVSo=
+github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
@@ -275,19 +279,27 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/ktrysmt/go-bitbucket v0.6.4 h1:C8dUGp0qkwncKtAnozHCbbqhptefzEd1I0sfnuy9rYQ=
+github.com/ktrysmt/go-bitbucket v0.6.4/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
 github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/markbates/pkger v0.15.1 h1:3MPelV53RnGSW07izx5xGxl4e/sdRD6zqseIk0rMASY=
 github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238 h1:+MZW2uvHgN8kYvksEN3f7eFL2wpzk0GxmlFsMybWc7E=
+github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mutecomm/go-sqlcipher/v4 v4.4.0 h1:sV1tWCWGAVlPhNGT95Q+z/txFxuhAYWwHD1afF5bMZg=
@@ -421,6 +433,7 @@ golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -456,6 +469,7 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgN
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4=
 golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -474,6 +488,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03i
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180224232135-f6cff0780e54/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -602,6 +617,7 @@ google.golang.org/api v0.29.0 h1:BaiDisFir8O4IJxvAabCGGkQ6yCJegNQqSVoYUNAnbk=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0 h1:yfrXXP61wVuLb0vBcG6qaOoIoqYEzOQS8jum51jkv2w=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

internal/cli/build_bitbucket.go

@@ -0,0 +1,7 @@
+// +build bitbucket
+
+package cli
+
+import (
+	_ "github.com/golang-migrate/migrate/v4/source/bitbucket"
+)

source/bitbucket/.gitignore

@@ -0,0 +1 @@
+.bitbucket_test_secrets

source/bitbucket/README.md

@@ -0,0 +1,14 @@
+# bitbucket
+
+This driver is catered for those that want to source migrations from bitbucket cloud(https://bitbucket.com).
+
+`bitbucket://user:password@owner/repo/path#ref`
+
+| URL Query  | WithInstance Config | Description |
+|------------|---------------------|-------------|
+| user | | The username of the user connecting |
+| password | | User's password or an app password with repo read permission  |
+| owner | | the repo owner |
+| repo | | the name of the repository |
+| path | | path in repo to migrations |
+| ref | | (optional) can be a SHA, branch, or tag |

source/bitbucket/bitbucket.go

@@ -0,0 +1,208 @@
+package bitbucket
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	nurl "net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/golang-migrate/migrate/v4/source"
+	"github.com/ktrysmt/go-bitbucket"
+)
+
+func init() {
+	source.Register("bitbucket", &Bitbucket{})
+}
+
+var (
+	ErrNoUserInfo             = fmt.Errorf("no username:password provided")
+	ErrNoAccessToken          = fmt.Errorf("no password/app password")
+	ErrInvalidRepo            = fmt.Errorf("invalid repo")
+	ErrInvalidBitbucketClient = fmt.Errorf("expected *bitbucket.Client")
+	ErrNoDir                  = fmt.Errorf("no directory")
+)
+
+type Bitbucket struct {
+	config     *Config
+	client     *bitbucket.Client
+	migrations *source.Migrations
+}
+
+type Config struct {
+	Owner string
+	Repo  string
+	Path  string
+	Ref   string
+}
+
+func (b *Bitbucket) Open(url string) (source.Driver, error) {
+	u, err := nurl.Parse(url)
+	if err != nil {
+		return nil, err
+	}
+
+	if u.User == nil {
+		return nil, ErrNoUserInfo
+	}
+
+	password, ok := u.User.Password()
+	if !ok {
+		return nil, ErrNoAccessToken
+	}
+
+	cl := bitbucket.NewBasicAuth(u.User.Username(), password)
+
+	cfg := &Config{}
+	// set owner, repo and path in repo
+	cfg.Owner = u.Host
+	pe := strings.Split(strings.Trim(u.Path, "/"), "/")
+	if len(pe) < 1 {
+		return nil, ErrInvalidRepo
+	}
+	cfg.Repo = pe[0]
+	if len(pe) > 1 {
+		cfg.Path = strings.Join(pe[1:], "/")
+	}
+	cfg.Ref = u.Fragment
+
+	bi, err := WithInstance(cl, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return bi, nil
+}
+
+func WithInstance(client *bitbucket.Client, config *Config) (source.Driver, error) {
+	bi := &Bitbucket{
+		client:     client,
+		config:     config,
+		migrations: source.NewMigrations(),
+	}
+
+	if err := bi.readDirectory(); err != nil {
+		return nil, err
+	}
+
+	return bi, nil
+}
+
+func (b *Bitbucket) readDirectory() error {
+	b.ensureFields()
+
+	fOpt := &bitbucket.RepositoryFilesOptions{
+		Owner:    b.config.Owner,
+		RepoSlug: b.config.Repo,
+		Ref:      b.config.Ref,
+		Path:     b.config.Path,
+	}
+
+	dirContents, err := b.client.Repositories.Repository.ListFiles(fOpt)
+
+	if err != nil {
+		return err
+	}
+
+	for _, fi := range dirContents {
+
+		m, err := source.DefaultParse(filepath.Base(fi.Path))
+		if err != nil {
+			continue // ignore files that we can't parse
+		}
+		if !b.migrations.Append(m) {
+			return fmt.Errorf("unable to parse file %v", fi.Path)
+		}
+	}
+
+	return nil
+}
+
+func (b *Bitbucket) ensureFields() {
+	if b.config == nil {
+		b.config = &Config{}
+	}
+}
+
+func (b *Bitbucket) Close() error {
+	return nil
+}
+
+func (b *Bitbucket) First() (version uint, er error) {
+	b.ensureFields()
+
+	if v, ok := b.migrations.First(); !ok {
+		return 0, &os.PathError{Op: "first", Path: b.config.Path, Err: os.ErrNotExist}
+	} else {
+		return v, nil
+	}
+}
+
+func (b *Bitbucket) Prev(version uint) (prevVersion uint, err error) {
+	b.ensureFields()
+
+	if v, ok := b.migrations.Prev(version); !ok {
+		return 0, &os.PathError{Op: fmt.Sprintf("prev for version %v", version), Path: b.config.Path, Err: os.ErrNotExist}
+	} else {
+		return v, nil
+	}
+}
+
+func (b *Bitbucket) Next(version uint) (nextVersion uint, err error) {
+	b.ensureFields()
+
+	if v, ok := b.migrations.Next(version); !ok {
+		return 0, &os.PathError{Op: fmt.Sprintf("next for version %v", version), Path: b.config.Path, Err: os.ErrNotExist}
+	} else {
+		return v, nil
+	}
+}
+
+func (b *Bitbucket) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) {
+	b.ensureFields()
+
+	if m, ok := b.migrations.Up(version); ok {
+		fBlobOpt := &bitbucket.RepositoryBlobOptions{
+			Owner:    b.config.Owner,
+			RepoSlug: b.config.Repo,
+			Ref:      b.config.Ref,
+			Path:     path.Join(b.config.Path, m.Raw),
+		}
+		file, err := b.client.Repositories.Repository.GetFileBlob(fBlobOpt)
+		if err != nil {
+			return nil, "", err
+		}
+		if file != nil {
+			r := file.Content
+			return ioutil.NopCloser(strings.NewReader(string(r))), m.Identifier, nil
+		}
+	}
+	return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: b.config.Path, Err: os.ErrNotExist}
+}
+
+func (b *Bitbucket) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) {
+	b.ensureFields()
+
+	if m, ok := b.migrations.Down(version); ok {
+		fBlobOpt := &bitbucket.RepositoryBlobOptions{
+			Owner:    b.config.Owner,
+			RepoSlug: b.config.Repo,
+			Ref:      b.config.Ref,
+			Path:     path.Join(b.config.Path, m.Raw),
+		}
+		file, err := b.client.Repositories.Repository.GetFileBlob(fBlobOpt)
+
+		if err != nil {
+			return nil, "", err
+		}
+		if file != nil {
+			r := file.Content
+
+			return ioutil.NopCloser(strings.NewReader(string(r))), m.Identifier, nil
+		}
+	}
+	return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: b.config.Path, Err: os.ErrNotExist}
+}

source/bitbucket/bitbucket_test.go

@@ -0,0 +1,33 @@
+package bitbucket
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	st "github.com/golang-migrate/migrate/v4/source/testing"
+)
+
+var BitbucketTestSecret = "" // username:password
+
+func init() {
+	secrets, err := ioutil.ReadFile(".bitbucket_test_secrets")
+	if err == nil {
+		BitbucketTestSecret = string(bytes.TrimSpace(secrets)[:])
+	}
+}
+
+func Test(t *testing.T) {
+	if len(BitbucketTestSecret) == 0 {
+		t.Skip("test requires .bitbucket_test_secrets")
+	}
+
+	b := &Bitbucket{}
+
+	d, err := b.Open("bitbucket://" + BitbucketTestSecret + "@abhishekbipp/test-migration/migrations/test#master")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st.Test(t, d)
+}