From c7d1059bc575e72dcc31d86808ddb0ebc742f9f5 Mon Sep 17 00:00:00 2001 From: Luke Hopkins Date: Sun, 2 Jun 2019 22:34:15 +0100 Subject: [PATCH] utilty to support downloading all repos in a github org or syncronize all repos in a directory --- .goreleaser.yml | 28 ++++ Makefile | 15 ++ README.md | 14 ++ cmd/github.go | 301 ++++++++++++++++++++++++++++++++++++ cmd/github_test.go | 174 +++++++++++++++++++++ cmd/local.go | 76 +++++++++ cmd/root.go | 56 +++++++ foobar.go.bak | 35 +++++ go.mod | 12 ++ go.sum | 159 +++++++++++++++++++ main.go | 21 +++ pkg/actions/actions.go | 222 ++++++++++++++++++++++++++ pkg/actions/actions_test.go | 126 +++++++++++++++ pkg/debug/debug.go | 18 +++ 14 files changed, 1257 insertions(+) create mode 100644 .goreleaser.yml create mode 100644 Makefile create mode 100644 README.md create mode 100644 cmd/github.go create mode 100644 cmd/github_test.go create mode 100644 cmd/local.go create mode 100644 cmd/root.go create mode 100644 foobar.go.bak create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 pkg/actions/actions.go create mode 100644 pkg/actions/actions_test.go create mode 100644 pkg/debug/debug.go diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..18d66df --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,28 @@ +# This is an example goreleaser.yaml file with some sane defaults. +# Make sure to check the documentation at http://goreleaser.com +before: + hooks: + # you may remove this if you don't use vgo + - go mod download + # you may remove this if you don't need go generate + - go generate ./... +builds: +- env: + - CGO_ENABLED=0 +archive: + replacements: + darwin: Darwin + linux: Linux + windows: Windows + 386: i386 + amd64: x86_64 +checksum: + name_template: 'checksums.txt' +snapshot: + name_template: "{{ .Tag }}-next" +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0d6dcb3 --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +test: + go test ./... + +test-cover: + go test ./... -coverprofile=coverage.out + go tool cover -html=coverage.out + rm coverage.out + +lint: + golangci-lint run + +release: + git tag -a $$VERSION + git push origin $$VERSION + goreleaser diff --git a/README.md b/README.md new file mode 100644 index 0000000..af9cf89 --- /dev/null +++ b/README.md @@ -0,0 +1,14 @@ +Tool to sync all repos for a github org or in a local directory + +### Install +`go get -u git@github.com:lhopki01/git-mass-sync` + +### Usage + +#### Sync all repos in a github org + +`git-mass-sync github kubernetes ~/github/kubernetes` + +#### Find all git repos in a local directory and run hub sync on them + +`git-mass-sync local ~/github/local_repos` diff --git a/cmd/github.go b/cmd/github.go new file mode 100644 index 0000000..8376bf8 --- /dev/null +++ b/cmd/github.go @@ -0,0 +1,301 @@ +// Copyright © 2019 NAME HERE +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/lhopki01/git-mass-sync/pkg/actions" + "github.com/lhopki01/git-mass-sync/pkg/debug" + "github.com/mitchellh/colorstring" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +type HttpClient interface { + Do(req *http.Request) (*http.Response, error) +} + +type action int + +const ( + actionClone action = iota + actionSync + actionArchive + actionCloneArchive + actionNone +) + +type repo struct { + SSHURL string `json:"ssh_url"` + Name string `json:"name"` + Archived bool `json:"archived"` +} + +// githubCmd represents the base command when called without any subcommands +var githubCmd = &cobra.Command{ + Use: "github [org] [download dir]", + Short: "Download all repos in a github org", + Run: func(cmd *cobra.Command, args []string) { + runGithub(args) + }, +} + +func init() { + rootCmd.AddCommand(githubCmd) + + githubCmd.Flags().String("regex", ".*", "Regex to match repo names against") + githubCmd.Flags().String("archive-dir", "", "Repo to put archived repos in\n(default is .archive in the download dir)") + + err := viper.BindPFlags(githubCmd.PersistentFlags()) + if err != nil { + log.Fatalf("Binding flags failed: %s", err) + } + viper.AutomaticEnv() +} + +func processFlags(args []string) (string, string, string, *regexp.Regexp) { + if len(args) != 2 { + log.Fatal("Wrong number of arguments") + } + org := args[0] + dir := filepath.Clean(args[1]) + + fmt.Println("=============") + fmt.Printf("Syncing org %s into %s\n", org, dir) + + archiveDir := viper.GetString("archive-Dir") + if archiveDir == "" { + archiveDir = fmt.Sprintf("%s/.archive", dir) + } else { + archiveDir = filepath.Clean(archiveDir) + } + fmt.Printf("Archiving repos into %s\n", archiveDir) + + r := regexp.MustCompile(viper.GetString("regex")) + + fmt.Println("=============") + + return dir, archiveDir, org, r +} + +func runGithub(args []string) { + dir, archiveDir, org, r := processFlags(args) + + client := &http.Client{} + repoList := getRepoList(org, client) + dirList := actions.GetGitDirList(dir) + + reposToSync, reposToClone, reposToArchive := repoActions(repoList, dirList, archiveDir, r) + + lenSync := len(reposToSync) + lenClone := len(reposToClone) + lenArchive := len(reposToArchive) + + fmt.Println("=============") + colorstring.Printf("[green]%d repos to sync\n", lenSync) + colorstring.Printf("[cyan]%d repos to clone\n", lenClone) + colorstring.Printf("[light_magenta]%d repos to archive\n", lenArchive) + fmt.Println("=============") + + // Order is very important here. Clone must always come before archive + failedSyncRepos, warningSyncRepos := actions.SyncRepos(reposToSync, dir) + failedCloneRepos := actions.CloneRepos(reposToClone, dir) + failedArchiveRepos := actions.ArchiveRepos(reposToArchive, dir, archiveDir) + + lenSyncWarnings := len(warningSyncRepos) + lenSyncFailures := len(failedSyncRepos) + lenCloneFailures := len(failedCloneRepos) + lenArchiveFailures := len(failedArchiveRepos) + + if lenSyncWarnings > 0 { + fmt.Println("=============") + //nolint:errcheck + colorstring.Println("[yellow]Warnings:") + for _, s := range warningSyncRepos { + colorstring.Printf(s) + } + } + if lenSyncFailures > 0 || lenCloneFailures > 0 || lenArchiveFailures > 0 { + fmt.Println("=============") + //nolint:errcheck + colorstring.Println("[red]Errors:") + for _, s := range failedSyncRepos { + //nolint:errcheck + colorstring.Println(s) + } + for _, s := range failedCloneRepos { + //nolint:errcheck + colorstring.Println(s) + } + for _, s := range failedArchiveRepos { + //nolint:errcheck + colorstring.Println(s) + } + } + + if !viper.GetBool("dry-run") { + fmt.Println("=============") + if lenSyncFailures > 0 { + colorstring.Printf( + "[red]%d[reset]/[green]%d repos synced\n", + lenSync-lenSyncFailures, + lenSync, + ) + } else if lenSync != 0 { + colorstring.Printf( + "[green]%d/%d repos synced\n", + lenSync-lenSyncFailures, + lenSync, + ) + } + if lenCloneFailures > 0 { + colorstring.Printf("[red]%d[reset]/[cyan]%d repos cloned\n", lenClone-lenCloneFailures, lenClone) + } else if lenClone != 0 { + colorstring.Printf("[cyan]%d/%d repos cloned\n", lenClone-lenCloneFailures, lenClone) + + } + if lenArchiveFailures > 0 { + colorstring.Printf("[red]%d[reset]/[light_magenta]%d repos archived\n", lenArchive-lenArchiveFailures, lenArchive) + } else if lenArchive != 0 { + colorstring.Printf("[light_magenta]%d/%d repos archived\n", lenArchive-lenArchiveFailures, lenArchive) + } + } +} + +func repoAction(repo repo, dirList []string) (action, []string) { + for i, dir := range dirList { + if dir == repo.Name { + if repo.Archived { + dirList = actions.RemoveElementFromSlice(dirList, i) + return actionArchive, dirList + } + dirList = actions.RemoveElementFromSlice(dirList, i) + return actionSync, dirList + } + } + if !repo.Archived { + return actionClone, dirList + } else if repo.Archived { + return actionCloneArchive, dirList + } + return actionNone, dirList +} + +func repoActions(repoList []repo, dirList []string, archiveDir string, r *regexp.Regexp) ([]string, []string, []string) { + var reposToSync []string + var reposToClone []string + var reposToArchive []string + + for _, repo := range repoList { + if r.MatchString(repo.Name) { + var a action + a, dirList = repoAction(repo, dirList) + switch a { + case actionArchive: + reposToArchive = append(reposToArchive, repo.Name) + continue + case actionSync: + reposToSync = append(reposToSync, repo.Name) + continue + case actionClone: + reposToClone = append(reposToClone, repo.SSHURL) + continue + case actionCloneArchive: + if _, err := os.Stat(fmt.Sprintf("%s/%s", archiveDir, repo.Name)); os.IsNotExist(err) { + reposToArchive = append(reposToArchive, repo.Name) + reposToClone = append(reposToClone, repo.SSHURL) + } + continue + } + } + } + reposToArchive = append(reposToArchive, dirList...) + + return reposToSync, reposToClone, reposToArchive +} + +func getRepoList(org string, client HttpClient) []repo { + fmt.Printf("Getting repo list") + + var repoList []repo + url := fmt.Sprintf("https://api.github.com/orgs/%s/repos?per_page=100", org) + //url := fmt.Sprintf("https://api.github.com/user/repos?per_page=100") + token := os.Getenv("GITHUB_TOKEN") + for url != "" { + fmt.Printf(".") + debug.Debug(url) + + req, _ := http.NewRequest("GET", url, nil) + req.Header.Add("Authorization", fmt.Sprintf("token %s", token)) + resp, err := client.Do(req) + if resp.StatusCode != 200 { + log.Fatalf("Unknown response %d for request: %s", resp.StatusCode, url) + } + if err != nil { + log.Fatalf("Github api request failed with err: %v", err) + } + + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(resp.Body) + if err != nil { + log.Fatalf("Failed to read repose body: %v", resp.Body) + } + + var repos []repo + err = json.Unmarshal(buf.Bytes(), &repos) + if err != nil { + fmt.Println(buf.String()) + fmt.Println(err) + } + repoList = append(repoList, repos...) + + url = getNextPageLink(resp.Header) + } + fmt.Println("") + return repoList +} + +func getNextPageLink(headers http.Header) (nextPage string) { + links, ok := headers["Link"] + if ok { + for _, link := range strings.Split(links[0], ",") { + segments := strings.Split(strings.TrimSpace(link), ";") + if len(segments) < 2 { + continue + } + if strings.TrimSpace(segments[1]) == `rel="next"` { + // check we have a real url between <> + url, err := url.Parse(segments[0][1 : len(segments[0])-1]) + if err != nil { + continue + } + return url.String() + } + } + } else { + return "" + } + return "" +} diff --git a/cmd/github_test.go b/cmd/github_test.go new file mode 100644 index 0000000..b74829e --- /dev/null +++ b/cmd/github_test.go @@ -0,0 +1,174 @@ +package cmd + +import ( + "bytes" + "io/ioutil" + "net/http" + "regexp" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNextPageLink(t *testing.T) { + type testCase struct { + tName string + linkHeader string + addLink bool + expectedNextPage string + } + testCases := []testCase{ + { + tName: "existing next page", + linkHeader: `; rel="next", ; rel="last"`, + addLink: true, + expectedNextPage: `https://api.github.com/organizations/16915932/repos?page=2`, + }, + { + tName: "no next page", + linkHeader: `; rel="prev", ; rel="first"`, + addLink: true, + expectedNextPage: "", + }, + { + tName: "no next links", + linkHeader: ``, + addLink: false, + expectedNextPage: "", + }, + } + for _, tc := range testCases { + tc := tc + t.Run(tc.tName, func(t *testing.T) { + t.Parallel() + h := http.Header{} + if tc.addLink { + h.Add("Link", tc.linkHeader) + } + assert.Equal(t, tc.expectedNextPage, getNextPageLink(h)) + }) + } +} + +func TestRepoActions(t *testing.T) { + type testCase struct { + tName string + repo repo + dirList []string + expectedAction action + expectedDirList []string + } + testCases := []testCase{ + { + tName: "repo to archive", + repo: repo{ + Name: "archivedRepo", + Archived: true, + SSHURL: "git@giturl", + }, + dirList: []string{"archivedRepo", "syncRepo", "deletedRepo"}, + expectedAction: actionArchive, + expectedDirList: []string{"syncRepo", "deletedRepo"}, + }, + { + tName: "repo to clone", + repo: repo{ + Name: "cloneRepo", + Archived: false, + SSHURL: "git@giturl/cloneRepo", + }, + dirList: []string{"archivedRepo", "syncRepo", "deletedRepo"}, + expectedAction: actionClone, + expectedDirList: []string{"archivedRepo", "syncRepo", "deletedRepo"}, + }, + { + tName: "repo to sync", + repo: repo{ + Name: "syncRepo", + Archived: false, + SSHURL: "git@giturl", + }, + dirList: []string{"archivedRepo", "syncRepo", "deletedRepo"}, + expectedAction: actionSync, + expectedDirList: []string{"archivedRepo", "deletedRepo"}, + }, + { + tName: "repo to clone and archive", + repo: repo{ + Name: "cloneArchiveRepo", + Archived: true, + SSHURL: "git@giturl/cloneArchiveRepo", + }, + dirList: []string{"archivedRepo", "syncRepo", "deletedRepo"}, + expectedAction: actionCloneArchive, + expectedDirList: []string{"archivedRepo", "syncRepo", "deletedRepo"}, + }, + } + var repos []repo + for _, tc := range testCases { + tc := tc + t.Run(tc.tName, func(t *testing.T) { + action, dirList := repoAction(tc.repo, tc.dirList) + assert.Equal(t, tc.expectedAction, action) + assert.Equal(t, tc.expectedDirList, dirList) + }) + repos = append(repos, tc.repo) + } + + r, _ := regexp.Compile(".*") + reposToSync, reposToClone, reposToArchive := repoActions(repos, []string{"archivedRepo", "syncRepo", "deletedRepo"}, "foobar", r) + assert.Equal(t, []string{"syncRepo"}, reposToSync) + assert.Equal(t, []string{"git@giturl/cloneRepo", "git@giturl/cloneArchiveRepo"}, reposToClone) + assert.Equal(t, []string{"archivedRepo", "cloneArchiveRepo", "deletedRepo"}, reposToArchive) +} + +type MockClient struct { + DoFunc func(req *http.Request) (*http.Response, error) +} + +func (m *MockClient) Do(req *http.Request) (*http.Response, error) { + if m.DoFunc != nil { + return m.DoFunc(req) + } + // just in case you want default correct return value + return &http.Response{}, nil +} + +func TestGetRepoList(t *testing.T) { + body := ioutil.NopCloser(bytes.NewReader([]byte(` + [ + { + "Name": "foobar", + "ssh_url": "git@github.com/foobar.git", + "Archive": false + } + ] + `))) + + client := &MockClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // do whatever you want + return &http.Response{ + StatusCode: http.StatusOK, + Body: body, + }, nil + }, + } + repoList := getRepoList("foobar", client) + expectRepos := []repo{ + { + SSHURL: "git@github.com/foobar.git", Name: "foobar", Archived: false, + }, + } + assert.Equal(t, expectRepos, repoList) + +} + +func TestProcessFlags(t *testing.T) { + dir, archiveDir, org, r := processFlags([]string{"foobar", "/tmp/foobar"}) + assert.Equal(t, "/tmp/foobar", dir) + assert.Equal(t, "/tmp/foobar/.archive", archiveDir) + assert.Equal(t, "foobar", org) + expectedR, _ := regexp.Compile("") + assert.Equal(t, expectedR, r) +} diff --git a/cmd/local.go b/cmd/local.go new file mode 100644 index 0000000..d816686 --- /dev/null +++ b/cmd/local.go @@ -0,0 +1,76 @@ +package cmd + +import ( + "fmt" + "path/filepath" + + "github.com/lhopki01/git-mass-sync/pkg/actions" + "github.com/mitchellh/colorstring" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var localCmd = &cobra.Command{ + Use: "local [target dir]", + Short: "Sync all repos within the target directory", + Args: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("Wrong number of arguments") + } + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + runLocal(args) + }, +} + +func init() { + rootCmd.AddCommand(localCmd) +} + +func runLocal(args []string) { + dir := filepath.Clean(args[0]) + + fmt.Println("=============") + fmt.Printf("Syncing all git repos in %s", dir) + fmt.Println("=============") + + reposToSync := actions.GetGitDirList(dir) + lenSync := len(reposToSync) + + fmt.Println("=============") + colorstring.Printf("[green]%d repos to sync\n", lenSync) + fmt.Println("=============") + + failedSyncRepos, warningSyncRepos := actions.SyncRepos(reposToSync, dir) + lenSyncWarnings := len(warningSyncRepos) + lenSyncFailures := len(failedSyncRepos) + + if lenSyncWarnings > 0 { + fmt.Println("=============") + //nolint:errcheck + colorstring.Println("[green]Sync repos [yellow]warnings") + for _, s := range warningSyncRepos { + //nolint:errcheck + colorstring.Println(s) + } + } + if lenSyncFailures > 0 { + fmt.Println("=============") + //nolint:errcheck + colorstring.Println("[red]Failed [green]sync repos") + for _, s := range failedSyncRepos { + //nolint:errcheck + colorstring.Println(s) + } + } + if !viper.GetBool("dry-run") { + fmt.Println("=============") + if lenSyncFailures > 0 { + colorstring.Printf("[red]%d[reset]/[green]%d repos synced", lenSync-lenSyncFailures, lenSync) + + } else { + colorstring.Printf("[green]%d/%d repos synced", lenSync-lenSyncFailures, lenSync) + } + } +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..6abe5ee --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,56 @@ +// Copyright © 2019 NAME HERE +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "log" + "os" + + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "git-mass-sync [org] [download dir]", + Short: "Utility to mass download all git repos", +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + if err := rootCmd.Execute(); err != nil { + log.Println(err) + os.Exit(1) + } +} + +func init() { + // Here you will define your flags and configuration settings. + // Cobra supports persistent flags, which, if defined here, + // will be global for your application. + + // Cobra also supports local flags, which will only run + // when this action is called directly. + rootCmd.PersistentFlags().BoolP("dry-run", "n", false, "Show what would happen") + rootCmd.PersistentFlags().BoolP("verbose", "v", false, "Make the operation more talkative") + rootCmd.PersistentFlags().Int("parallelism", 50, "Max parallel processes to run") + + err := viper.BindPFlags(rootCmd.PersistentFlags()) + if err != nil { + log.Fatalf("Binding flags failed: %s", err) + } + viper.AutomaticEnv() +} diff --git a/foobar.go.bak b/foobar.go.bak new file mode 100644 index 0000000..bb09132 --- /dev/null +++ b/foobar.go.bak @@ -0,0 +1,35 @@ +package main + +import ( + "time" + + "github.com/remeh/sizedwaitgroup" + "github.com/vbauerster/mpb" +) + +func main() { + testSlice := makeRange(1, 100) + swg := sizedwaitgroup.New(20) + p := mpb.New(mpb.WithWidth(64)) + bar := p.AddBar(int64(len(testSlice))) + //bar.Format("|#-|") + for _, test := range testSlice { + swg.Add() + go func(test int, swg *sizedwaitgroup.SizedWaitGroup, bar *mpb.Bar) { + //fmt.Printf("starting %d\n", test) + time.Sleep(1 * time.Second) + //fmt.Printf("finishing %d\n", test) + bar.Increment() + swg.Done() + }(test, &swg, bar) + } + swg.Wait() +} + +func makeRange(min, max int) []int { + a := make([]int, max-min+1) + for i := range a { + a[i] = min + i + } + return a +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7eec5b7 --- /dev/null +++ b/go.mod @@ -0,0 +1,12 @@ +module github.com/lhopki01/git-mass-sync + +go 1.12 + +require ( + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db + github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce + github.com/schollz/progressbar/v2 v2.13.0 + github.com/spf13/cobra v0.0.4 + github.com/spf13/viper v1.4.0 + github.com/stretchr/testify v1.3.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..b79bace --- /dev/null +++ b/go.sum @@ -0,0 +1,159 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce h1:aP+C+YbHZfOQlutA4p4soHi7rVUqHQdWEVMSkHfDTqY= +github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/schollz/progressbar/v2 v2.13.0 h1:phYVXliSjdTqGyCBg08b8JmgdFQYbMM4rFEQ9jYZsR0= +github.com/schollz/progressbar/v2 v2.13.0/go.mod h1:fBI3onORwtNtwCWJHsrXtjE3QnJOtqIZrvr3rDaF7L0= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.4 h1:S0tLZ3VOKl2Te0hpq8+ke0eSJPfCnNTPiDlsfwi1/NE= +github.com/spf13/cobra v0.0.4/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go new file mode 100644 index 0000000..c89d531 --- /dev/null +++ b/main.go @@ -0,0 +1,21 @@ +// Copyright © 2019 NAME HERE +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "github.com/lhopki01/git-mass-sync/cmd" + +func main() { + cmd.Execute() +} diff --git a/pkg/actions/actions.go b/pkg/actions/actions.go new file mode 100644 index 0000000..afd4249 --- /dev/null +++ b/pkg/actions/actions.go @@ -0,0 +1,222 @@ +package actions + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "strings" + + "github.com/lhopki01/git-mass-sync/pkg/debug" + "github.com/mitchellh/colorstring" + "github.com/remeh/sizedwaitgroup" + "github.com/schollz/progressbar/v2" + "github.com/spf13/viper" +) + +func SyncRepos(reposToSync []string, dir string) ([]string, []string) { + verbose := viper.GetBool("verbose") + dryRun := viper.GetBool("dry-run") + + swg := sizedwaitgroup.New(viper.GetInt("parallelism")) + + failureChannel := make(chan string) + doneFailure := make(chan bool) + warningChannel := make(chan string) + doneWarning := make(chan bool) + var failures []string + var warnings []string + go collectMessages(&failures, failureChannel, doneFailure) + go collectMessages(&warnings, warningChannel, doneWarning) + + bar := progressbar.NewOptions( + len(reposToSync), + progressbar.OptionEnableColorCodes(true), + progressbar.OptionShowCount(), + progressbar.OptionSetDescription("[green]Syncing repos"), + ) + + if verbose || dryRun { + //nolint:errcheck + colorstring.Println("[green]Syncing repos") + } else { + err := bar.RenderBlank() + if err != nil { + fmt.Printf("Can't render progress bar") + } + } + + for _, repo := range reposToSync { + if dryRun { + colorstring.Printf("[green]Would sync %s\n", repo) + } else { + swg.Add() + if verbose { + colorstring.Printf("[green]Syncing %s\n", repo) + } + go syncRepo(dir, repo, &swg, failureChannel, warningChannel, bar) + } + } + + swg.Wait() + close(doneFailure) + close(doneWarning) + <-doneFailure + <-doneWarning + + if !verbose || dryRun { + err := bar.Finish() + if err != nil { + fmt.Printf("Can't render progress bar finish") + } + println("") + } + + return failures, warnings +} + +func syncRepo(dir string, repo string, swg *sizedwaitgroup.SizedWaitGroup, failureChannel chan string, warningChannel chan string, bar *progressbar.ProgressBar) { + + cmd := exec.Command("hub", "sync") + cmd.Dir = fmt.Sprintf("%s/%s", dir, repo) + output, err := cmd.CombinedOutput() + debug.Debugf("Output of hub sync %s: %s", repo, string(output)) + if strings.Contains(string(output), "warning: ") { + warningChannel <- fmt.Sprintf("[green]Syncing %s: [yellow]%s", repo, string(output)) + } + if err != nil { + failureChannel <- fmt.Sprintf("[green]Syncing %s: [red]%s\n%s", repo, err, string(output)) + } + if !viper.GetBool("verbose") { + err := bar.Add(1) + if err != nil { + fmt.Printf("Can't add to progress bar") + } + } + swg.Done() +} + +func collectMessages(p *[]string, channel chan string, done chan bool) { + for msg := range channel { + *p = append(*p, msg) + } + done <- true +} + +func CloneRepos(reposToClone []string, dir string) []string { + swg := sizedwaitgroup.New(viper.GetInt("parallelism")) + + failureChannel := make(chan string) + done := make(chan bool) + var failures []string + go collectMessages(&failures, failureChannel, done) + + for _, repo := range reposToClone { + if viper.GetBool("dry-run") { + colorstring.Printf("[cyan]Would clone %s\n", repo) + } else { + swg.Add() + colorstring.Printf("[cyan]Cloning %s\n", repo) + go cloneRepo(dir, repo, &swg, failureChannel) + } + } + swg.Wait() + close(failureChannel) + <-done + return failures +} + +func cloneRepo(dir string, repo string, swg *sizedwaitgroup.SizedWaitGroup, failureChannel chan string) { + defer swg.Done() + cmd := exec.Command("git", "clone", repo) + cmd.Dir = dir + output, err := cmd.CombinedOutput() + debug.Debugf("Output of git clone %s: %s", repo, output) + if err != nil { + failureChannel <- fmt.Sprintf("[cyan]Cloning %s: [red]%s\n%s", repo, err, string(output)) + } +} + +func ArchiveRepos(reposToArchive []string, dir string, archiveDir string) []string { + swg := sizedwaitgroup.New(viper.GetInt("parallelism")) + + failureChannel := make(chan string) + done := make(chan bool) + var failures []string + go collectMessages(&failures, failureChannel, done) + + if _, err := os.Stat(archiveDir); os.IsNotExist(err) { + if viper.GetBool("dry-run") { + fmt.Printf("Would create archive dir %s if not exists\n", archiveDir) + } else { + fmt.Printf("Creating archiveDir %s", archiveDir) + err := os.MkdirAll(archiveDir, 0755) + if err != nil { + //nolint:errcheck + colorstring.Println("[red]Failed to create archive dir") + os.Exit(1) + } + } + } + for _, repo := range reposToArchive { + if viper.GetBool("dry-run") { + colorstring.Printf("[light_magenta]Would archive %s in %s\n", repo, archiveDir) + } else { + swg.Add() + colorstring.Printf("[light_magenta]Archiving %s in %s\n", repo, archiveDir) + go func(dir string, repo string, swg *sizedwaitgroup.SizedWaitGroup) { + defer swg.Done() + + err := os.Rename( + fmt.Sprintf("%s/%s", dir, repo), + fmt.Sprintf("%s/%s", archiveDir, repo), + ) + if err != nil { + failureChannel <- fmt.Sprintf("[light_magenta]Archiving %s: [red]%s\n", repo, err) + } + }(dir, repo, &swg) + } + } + swg.Wait() + close(failureChannel) + <-done + return failures +} + +func GetGitDirList(dir string) []string { + fmt.Printf("Getting existing git directory list") + var dirList []string + files, err := ioutil.ReadDir(dir) + if err != nil { + log.Fatal(err) + } + for i, f := range files { + if i%100 == 0 { + fmt.Printf(".") + } + if f.IsDir() { + cmd := exec.Command("git", "rev-parse") + cmd.Dir = fmt.Sprintf("%s/%s", dir, f.Name()) + err = cmd.Run() + if err == nil { + dirList = append(dirList, f.Name()) + } else { + debug.Debugf("%s is not a git directory", f.Name()) + } + } else { + debug.Debugf("%s is not a directory", f.Name()) + } + } + fmt.Println("") + return dirList +} + +func RemoveElementFromSlice(s []string, i int) []string { + // Does not preserve order + if len(s) <= i { + return s + } + s[i] = s[0] + return s[1:] +} diff --git a/pkg/actions/actions_test.go b/pkg/actions/actions_test.go new file mode 100644 index 0000000..99aa87d --- /dev/null +++ b/pkg/actions/actions_test.go @@ -0,0 +1,126 @@ +package actions + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "sort" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSyncRepos(t *testing.T) { + testDir := CreateTestDirs() + failures, warnings := SyncRepos([]string{"gitDir"}, testDir) + expectedFailures := []string{"[green]Syncing gitDir: [red]exit status 1\nno git remotes found\n"} + var expectedWarnings []string + assert.Equal(t, expectedFailures, failures) + assert.Equal(t, expectedWarnings, warnings) + os.RemoveAll(testDir) +} + +func TestCloneRepos(t *testing.T) { + testDir := CreateTestDirs() + failures := CloneRepos([]string{"git@gitub.com/foo/bar.git"}, testDir) + expectedFailures := []string{"[cyan]Cloning git@gitub.com/foo/bar.git: [red]exit status 128\nfatal: repository 'git@gitub.com/foo/bar.git' does not exist\n"} + assert.Equal(t, expectedFailures, failures) + os.RemoveAll(testDir) +} + +func TestArchiveRepos(t *testing.T) { + testDir := CreateTestDirs() + archiveDir := testDir + "/.archive" + failures := ArchiveRepos([]string{"gitDir", "nonExistantDir"}, testDir, archiveDir) + expectedFailures := []string{fmt.Sprintf("[light_magenta]Archiving nonExistantDir: [red]rename %s/nonExistantDir %s/.archive/nonExistantDir: no such file or directory\n", testDir, testDir)} + assert.Equal(t, expectedFailures, failures) + assert.DirExists(t, archiveDir+"/gitDir") + os.RemoveAll(testDir) +} + +func CreateTestDirs() string { + dir, err := ioutil.TempDir("", "git-mass-sync") + if err != nil { + log.Fatal(err) + } + + err = os.Mkdir(dir+"/gitDir", 0755) + if err != nil { + log.Fatal(err) + } + cmd := exec.Command("git", "init") + cmd.Dir = dir + "/gitDir" + err = cmd.Run() + if err != nil { + log.Fatal(err) + } + + err = os.Mkdir(dir+"/notGitDir", 0755) + if err != nil { + log.Fatal(err) + } + + _, err = os.Create(dir + "/file") + if err != nil { + log.Fatal(err) + } + return dir +} + +func TestGetDirList(t *testing.T) { + testDir := CreateTestDirs() + assert.Equal(t, []string{"gitDir"}, GetGitDirList(testDir)) + os.RemoveAll(testDir) +} + +func TestRemoveElementFromSlice(t *testing.T) { + type testCase struct { + tName string + slice []string + indexToRemove int + expectedSlice []string + } + testCases := []testCase{ + { + tName: "remove from the middle", + slice: []string{"a", "b", "c"}, + indexToRemove: 1, + expectedSlice: []string{"a", "c"}, + }, + { + tName: "remove from end", + slice: []string{"a", "b", "c"}, + indexToRemove: 2, + expectedSlice: []string{"a", "b"}, + }, + { + tName: "remove from beginning", + slice: []string{"a", "b", "c"}, + indexToRemove: 0, + expectedSlice: []string{"b", "c"}, + }, + { + tName: "remove from len1 slice", + slice: []string{"a"}, + indexToRemove: 0, + expectedSlice: []string{}, + }, + { + tName: "remove out of bounds index", + slice: []string{"a", "b", "c"}, + indexToRemove: 3, + expectedSlice: []string{"a", "b", "c"}, + }, + } + for _, tc := range testCases { + tc := tc + t.Run(tc.tName, func(t *testing.T) { + t.Parallel() + result := RemoveElementFromSlice(tc.slice, tc.indexToRemove) + sort.Strings(result) + assert.Equal(t, tc.expectedSlice, result) + }) + } +} diff --git a/pkg/debug/debug.go b/pkg/debug/debug.go new file mode 100644 index 0000000..701ec39 --- /dev/null +++ b/pkg/debug/debug.go @@ -0,0 +1,18 @@ +package debug + +import ( + "fmt" + "os" + + "github.com/spf13/viper" +) + +func Debugf(format string, a ...interface{}) { + Debug(fmt.Sprintf(format, a...)) +} + +func Debug(a ...interface{}) { + if viper.GetBool("verbose") { + fmt.Fprintln(os.Stderr, a...) + } +}