diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 3532ac0..d193bbe 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -13,10 +13,10 @@ jobs: - name: Checkout code uses: actions/checkout@v1 - name: Run unittest - run: make test + run: cd baetyl-remote-object && make test - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: token: ${{ secrets.CODECOV_TOKEN }} - file: ./coverage.txt + file: ./baetyl-remote-object/coverage.txt fail_ci_if_error: true \ No newline at end of file diff --git a/.github/workflows/image-develop.yml b/.github/workflows/image-develop.yml new file mode 100644 index 0000000..1167f03 --- /dev/null +++ b/.github/workflows/image-develop.yml @@ -0,0 +1,30 @@ +name: image-develop +on: + push: + branches: + - master + +jobs: + image: + name: docker build images + runs-on: ubuntu-latest + steps: + - name: Install deps + run: sudo apt update -y && sudo apt install -y qemu qemu-user-static + - name: Install Docker CE for buildx + run: | + sudo apt update + sudo apt install apt-transport-https ca-certificates curl gnupg-agent software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + sudo apt update + sudo apt install docker-ce + docker -v + - name: Checkout code + uses: actions/checkout@v1 + - name: docker login + run: | + docker login -u ${{ secrets.DOCKER_REGISTRY_ID }} -p ${{ secrets.DOCKER_REGISTRY_PASS }} + - name: build and publish image + run: | + cd baetyl-remote-object && make image PLATFORMS=all XFLAGS='--push --cache-to=type=local,dest=/tmp/main' REGISTRY=baetyltechtest/ diff --git a/.github/workflows/image-release.yml b/.github/workflows/image-release.yml new file mode 100644 index 0000000..ba23883 --- /dev/null +++ b/.github/workflows/image-release.yml @@ -0,0 +1,31 @@ +name: image-release +on: + push: + tags: + - 'v*' + +jobs: + image: + name: docker build images + runs-on: ubuntu-latest + steps: + - name: Install deps + run: sudo apt update -y && sudo apt install -y qemu qemu-user-static + - name: Install Docker CE for buildx + run: | + sudo apt update + sudo apt install apt-transport-https ca-certificates curl gnupg-agent software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + sudo apt update + sudo apt install docker-ce + docker -v + - name: Checkout code + uses: actions/checkout@v1 + - name: docker login + run: | + docker login -u ${{ secrets.DOCKER_REGISTRY_ID }} -p ${{ secrets.DOCKER_REGISTRY_PASS }} + - name: build and publish image + run: | + cd baetyl-remote-object && make image PLATFORMS=all XFLAGS='--push --cache-to=type=local,dest=/tmp/main' REGISTRY=baetyltech/ + \ No newline at end of file diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 5d75e4c..4f51d17 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -13,4 +13,4 @@ jobs: - name: Checkout code uses: actions/checkout@v1 - name: Run unittest - run: make test \ No newline at end of file + run: cd baetyl-remote-object && make test \ No newline at end of file diff --git a/.gitignore b/.gitignore index 11e036d..97b2331 100644 --- a/.gitignore +++ b/.gitignore @@ -4,13 +4,34 @@ *.dll *.so *.dylib -/**/coverage.out -/**/coverage.html +/.idea +/.vscode +/**/coverage.* /**/.DS_Store -/output - +/**/*.log +/vendor # Test binary, build with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out + +# Other +*~ +*.swp +*.a +*.jar +*.iml +/.idea +/.vscode +/output* +/**/*.zip +/**/.DS_Store +/**/*.db +/**/*.db.lock +/**/*debug.test +/**/debug +/**/*.log +baetyl-remote-object +/**/baetyl-remote-object +/**/output/ diff --git a/Makefile b/Makefile deleted file mode 100644 index cac0bd2..0000000 --- a/Makefile +++ /dev/null @@ -1,55 +0,0 @@ -MODULES?=object -PLATFORM_ALL:=darwin/amd64 linux/amd64 linux/arm64 linux/386 linux/arm/v7 linux/arm/v6 linux/arm/v5 linux/ppc64le linux/s390x - -GIT_REV:=git-$(shell git rev-parse --short HEAD) -GIT_TAG:=$(shell git tag --contains HEAD) -VERSION:=$(if $(GIT_TAG),$(GIT_TAG),$(GIT_REV)) - -GO_OS:=$(shell go env GOOS) -GO_ARCH:=$(shell go env GOARCH) -GO_ARM:=$(shell go env GOARM) -GO_TEST_FLAGS?= -GO_TEST_PKGS?=$(shell go list ./...) - -ifndef PLATFORMS - GO_OS:=$(shell go env GOOS) - GO_ARCH:=$(shell go env GOARCH) - GO_ARM:=$(shell go env GOARM) - PLATFORMS:=$(if $(GO_ARM),$(GO_OS)/$(GO_ARCH)/$(GO_ARM),$(GO_OS)/$(GO_ARCH)) - ifeq ($(GO_OS),darwin) - PLATFORMS+=linux/amd64 - endif -else ifeq ($(PLATFORMS),all) - override PLATFORMS:=$(PLATFORM_ALL) -endif - -OUTPUT:=output - -OUTPUT_MODS:=$(MODULES:%=baetyl-remote-%) -TEST_MODS:=$(MODULES:%=test/baetyl-remote-%) -IMAGE_MODS:=$(MODULES:%=image/baetyl-remote-%) - -.PHONY: all $(OUTPUT_MODS) -all: $(OUTPUT_MODS) - -$(OUTPUT_MODS): - @make -C $@ - -.PHONY: image $(IMAGE_MODS) -image: $(IMAGE_MODS) - -$(IMAGE_MODS): - @make -C $(notdir $@) image - -$(TEST_MODS): - @make -C $(notdir $@) test - -.PHONY: test -test: $(TEST_MODS) - -.PHONY: rebuild -rebuild: clean all - -.PHONY: clean -clean: - @-rm -rf $(OUTPUT) diff --git a/baetyl-remote-object/Dockerfile b/baetyl-remote-object/Dockerfile index 65e939f..d79ec33 100644 --- a/baetyl-remote-object/Dockerfile +++ b/baetyl-remote-object/Dockerfile @@ -1,9 +1,9 @@ -# Code generated by github.com/baetyl/baetyl/sdk/baetyl-go/generate.go. DO NOT EDIT. -# source: github.com/baetyl/baetyl/sdk/baetyl-go/templates/Dockerfile-go +FROM --platform=$TARGETPLATFORM golang:1.13.5-stretch as devel +ARG BUILD_ARGS +COPY / /go/src/ +RUN cd /go/src/baetyl-remote-object && make build-local BUILD_ARGS=$BUILD_ARGS FROM --platform=$TARGETPLATFORM busybox -ARG TARGETPLATFORM -ARG MODULE=baetyl-remote-object -COPY $TARGETPLATFORM/$MODULE/*.pem /etc/ssl/certs/ -COPY $TARGETPLATFORM/$MODULE/bin/$MODULE /bin/ +COPY --from=devel /go/src/baetyl-remote-object/pem/*.pem /etc/ssl/certs/ +COPY --from=devel /go/src/baetyl-remote-object/baetyl-remote-object /bin/ ENTRYPOINT ["baetyl-remote-object"] diff --git a/baetyl-remote-object/Dockerfile-local b/baetyl-remote-object/Dockerfile-local new file mode 100644 index 0000000..9d2da44 --- /dev/null +++ b/baetyl-remote-object/Dockerfile-local @@ -0,0 +1,3 @@ +FROM busybox +COPY baetyl-remote-object /bin/ +ENTRYPOINT ["baetyl-remote-object"] \ No newline at end of file diff --git a/baetyl-remote-object/Makefile b/baetyl-remote-object/Makefile index 66a2dd4..193c494 100644 --- a/baetyl-remote-object/Makefile +++ b/baetyl-remote-object/Makefile @@ -1,73 +1,97 @@ -# Code generated by github.com/baetyl/baetyl/sdk/baetyl-go/generate.go. DO NOT EDIT. -# source: github.com/baetyl/baetyl/sdk/baetyl-go/templates/Makefile-go - -MODULE:=baetyl-remote-object +MODULE:=remote-object +BIN:=baetyl-$(MODULE) SRC_FILES:=$(shell find . -type f -name '*.go') -PEM_FILES:=$(shell find pem -type f -name '*.pem' | sed 's/ /\\ /g') -BIN_FILE:=baetyl-remote-object -BIN_CMD=$(shell echo $(@:$(OUTPUT)/%/$(MODULE)/bin/$(BIN_FILE)=%) | sed 's:/v:/:g' | awk -F '/' '{print "CGO_ENABLED=0 GOOS="$$1" GOARCH="$$2" GOARM="$$3" go build"}') -o $@ ${GO_FLAGS} . -COPY_DIR:=../output -PLATFORM_ALL:=darwin/amd64 linux/amd64 linux/arm64 linux/386 linux/arm/v7 linux/arm/v6 linux/arm/v5 linux/ppc64le linux/s390x +PLATFORM_ALL:=darwin/amd64 linux/amd64 linux/arm64 linux/arm/v7 + +export DOCKER_CLI_EXPERIMENTAL=enabled -GIT_TAG:=$(shell git tag --contains HEAD) +GIT_TAG:=$(shell git tag --contains HEAD|awk 'END {print}') GIT_REV:=git-$(shell git rev-parse --short HEAD) VERSION:=$(if $(GIT_TAG),$(GIT_TAG),$(GIT_REV)) -GO_TEST_FLAGS?= -GO_TEST_PKGS?=$(shell go list ./...) +ifeq ($(findstring race,$(BUILD_ARGS)),race) +VERSION:=$(VERSION)-race +endif + +GO_OS:=$(shell go env GOOS) +GO_ARCH:=$(shell go env GOARCH) +GO_ARM:=$(shell go env GOARM) ifndef PLATFORMS - GO_OS:=$(shell go env GOOS) - GO_ARCH:=$(shell go env GOARCH) - GO_ARM:=$(shell go env GOARM) - PLATFORMS:=$(if $(GO_ARM),$(GO_OS)/$(GO_ARCH)/$(GO_ARM),$(GO_OS)/$(GO_ARCH)) - ifeq ($(GO_OS),darwin) - PLATFORMS+=linux/amd64 - endif + PLATFORMS:=$(if $(GO_ARM),$(GO_OS)/$(GO_ARCH)/$(GO_ARM),$(GO_OS)/$(GO_ARCH)) + ifeq ($(GO_OS),darwin) + PLATFORMS+=linux/amd64 + endif else ifeq ($(PLATFORMS),all) - override PLATFORMS:=$(PLATFORM_ALL) + override PLATFORMS:=$(PLATFORM_ALL) endif -REGISTRY?= -XFLAGS?=--load -XPLATFORMS:=$(shell echo $(filter-out darwin/amd64,$(PLATFORMS)) | sed 's: :,:g') -YML_FILE=$(wildcard *.yml) +GO := go +GO_MOD := $(GO) mod +GO_ENV := env GO111MODULE=on GOPROXY=https://goproxy.cn CGO_ENABLED=0 +GO_FLAGS := $(BUILD_ARGS) -ldflags '-X "github.com/baetyl/baetyl-go/v2/utils.REVISION=$(GIT_REV)" -X "github.com/baetyl/baetyl-go/v2/utils.VERSION=$(VERSION)"' +ifeq ($(findstring race,$(BUILD_ARGS)),race) +GO_ENV := env GO111MODULE=on GOPROXY=https://goproxy.cn CGO_ENABLED=1 +GO_FLAGS := $(BUILD_ARGS) -ldflags '-s -w -X "github.com/baetyl/baetyl-go/v2/utils.REVISION=$(GIT_REV)" -X "github.com/baetyl/baetyl-go/v2/utils.VERSION=$(VERSION)" -linkmode external -w -extldflags "-static"' +override PLATFORMS:= $(filter-out linux/arm/v7,$(PLATFORMS)) +endif +GO_BUILD := $(GO_ENV) $(GO) build $(GO_FLAGS) +GOTEST := $(GO) test +GOPKGS := $$($(GO) list ./... | grep -vE "vendor") + +REGISTRY:= +XFLAGS:=--load +XPLATFORMS:=$(shell echo $(filter-out darwin/amd64,$(PLATFORMS)) | sed 's: :,:g') -OUTPUT:=../output -OUTPUT_MODS:=$(PLATFORMS:%=$(OUTPUT)/%/$(MODULE)) -OUTPUT_BINS:=$(OUTPUT_MODS:%=%/bin/$(BIN_FILE)) -OUTPUT_PKGS:=$(OUTPUT_MODS:%=%/$(MODULE)-$(VERSION).zip) # TODO: switch to tar +OUTPUT :=output +OUTPUT_DIRS:=$(PLATFORMS:%=$(OUTPUT)/%/$(BIN)) +OUTPUT_BINS:=$(OUTPUT_DIRS:%=%/$(BIN)) +PKG_PLATFORMS := $(shell echo $(PLATFORMS) | sed 's:/:-:g') +OUTPUT_PKGS:=$(PKG_PLATFORMS:%=$(OUTPUT)/$(BIN)_%_$(VERSION).zip) .PHONY: all -all: $(OUTPUT_BINS) $(OUTPUT_PKGS) +all: build test + +.PHONY: build +build: $(OUTPUT_BINS) $(OUTPUT_BINS): $(SRC_FILES) @echo "BUILD $@" - @install -d -m 0755 $(dir $@) - $(BIN_CMD) + @mkdir -p $(dir $@) + @cp program.yml $(dir $@) + @$(shell echo $(@:$(OUTPUT)/%/$(BIN)/$(BIN)=%) | sed 's:/v:/:g' | awk -F '/' '{print "GOOS="$$1" GOARCH="$$2" GOARM="$$3""}') $(GO_BUILD) -o $@ . -$(OUTPUT_PKGS): $(OUTPUT_BINS) $(YML_FILE) $(PEM_FILES) - @echo "PACKAGE $@" - @install -m 0755 $(YML_FILE) $(dir $@) - @install -m 0755 $(PEM_FILES) $(dir $@) - @cd $(dir $@) && zip -q -r $(notdir $@) bin $(YML_FILE) +.PHONY: build-local +build-local: $(SRC_FILES) + @echo "BUILD $(BIN)" + $(GO_BUILD) -o $(BIN) . + @chmod +x $(BIN) .PHONY: image -image: $(OUTPUT_BINS) +image: @echo "BUILDX: $(REGISTRY)$(MODULE):$(VERSION)" @-docker buildx create --name baetyl @docker buildx use baetyl - docker buildx build $(XFLAGS) --platform $(XPLATFORMS) -t $(REGISTRY)$(MODULE):$(VERSION) -f Dockerfile $(COPY_DIR) + @docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker buildx build $(XFLAGS) --platform $(XPLATFORMS) -t $(REGISTRY)$(MODULE):$(VERSION) --build-arg BUILD_ARGS=$(BUILD_ARGS) -f Dockerfile .. + +.PHONY: test +test: fmt + $(GOTEST) -race -short -covermode=atomic -coverprofile=coverage.txt $(GOPKGS) + @go tool cover -func=coverage.txt | grep total -.PHONY: rebuild -rebuild: clean all +.PHONY: fmt +fmt: + $(GO_MOD) tidy + @go fmt ./... .PHONY: clean clean: - @find $(OUTPUT) -type d -name $(MODULE) | xargs rm -rf + @rm -rf $(OUTPUT) $(BIN) -.PHONY: test -test: - @go test ${GO_TEST_FLAGS} -coverprofile=coverage.out ${GO_TEST_PKGS} - @go tool cover -func=coverage.out | grep total - @go tool cover -html=coverage.out -o coverage.html +.PHONY: package +package: build $(OUTPUT_PKGS) + +$(OUTPUT_PKGS): + @echo "PACKAGE $@" + @cd $(OUTPUT)/$(shell echo $(@:$(OUTPUT)/$(BIN)_%_$(VERSION).zip=%) | sed 's:-:/:g')/$(BIN) && zip -q -r $(notdir $@) $(BIN) program.yml diff --git a/baetyl-remote-object/client.go b/baetyl-remote-object/client.go index 18eff13..f168eb7 100644 --- a/baetyl-remote-object/client.go +++ b/baetyl-remote-object/client.go @@ -1,26 +1,317 @@ package main import ( - "io" + "compress/flate" + "os" + "path" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/baetyl/baetyl-go/v2/errors" + "github.com/baetyl/baetyl-go/v2/log" + "github.com/baetyl/baetyl-go/v2/utils" + "github.com/docker/distribution/uuid" + "github.com/mholt/archiver" + "github.com/panjf2000/ants" ) -// CallAsync interface -type CallAsync func(msg *EventMessage, cb func(msg *EventMessage, err error)) error +type ruleHook func(msg *EventMessage, err error) + +// Task StorageClient +type Task struct { + msg *EventMessage + cb ruleHook +} + +// FileStats upload stats +type FileStats struct { + success uint64 + fail uint64 + limit uint64 + deleted uint64 +} + +// Client ObjectClient +type Client struct { + cfg ClientInfo + handler StorageHandler + stats Stats + pwd string + fs *FileStats + arch archiver.Archiver + log *log.Logger + pool *ants.PoolWithFunc + lock sync.Mutex + tomb utils.Tomb +} + +// NewClient creates a new object client +func NewClient(cfg ClientInfo) (*Client, error) { + pwd, err := os.Getwd() + if err != nil { + return nil, errors.Trace(err) + } + handler, err := NewObjectStorageHandler(cfg) + if err != nil { + return nil, errors.Trace(err) + } + cli := &Client{ + cfg: cfg, + pwd: pwd, + handler: handler, + fs: &FileStats{}, + log: log.With(log.Any("client", cfg.Name)), + } -// Start interface -type Start func() error + err = os.MkdirAll(cfg.TempPath, 0755) + if err != nil { + return nil, errors.Errorf("failed to make dir (%s): %s", cli.cfg.TempPath, err) + } + if ok := utils.FileExists(cli.cfg.Limit.Path); !ok { + basepath := path.Dir(cli.cfg.Limit.Path) + err = os.MkdirAll(basepath, 0755) + if err != nil { + return nil, errors.Errorf("failed to make dir (%s): %s", basepath, err) + } + f, err := os.Create(cli.cfg.Limit.Path) + if err != nil { + return nil, errors.Errorf("failed to make file (%s): %s", cli.cfg.Limit.Path, err.Error()) + } + defer f.Close() + } + err = utils.LoadYAML(cli.cfg.Limit.Path, &cli.stats) + if err != nil { + return nil, errors.Trace(err) + } + p, err := ants.NewPoolWithFunc(cli.cfg.Pool.Worker, cli.call, ants.WithExpiryDuration(cli.cfg.Pool.Idletime)) + if err != nil { + return nil, errors.Errorf("failed to create a pool: %s", err.Error()) + } + cli.pool = p -// Report reports stats -type report func(map[string]interface{}) error + cli.tomb.Go(cli.recording) + cli.log.Debug("client starts") + return cli, nil +} + +// CallAsync submit task +func (cli *Client) CallAsync(msg *EventMessage, cb ruleHook) error { + if cli.pool.Running() == cli.cfg.Pool.Worker { + cb(msg, errors.New("failed to submit task: no worker can be used")) + return nil + } + task := &Task{ + msg: msg, + cb: cb, + } + if err := cli.pool.Invoke(task); err != nil { + cb(msg, errors.Errorf("failed to invoke pool task: %s", err.Error())) + return nil + } + return nil +} + +func (cli *Client) call(task interface{}) { + t, ok := task.(*Task) + if !ok { + log.Error(errors.New("failed to convert interface{} to *Task")) + return + } + var err error + switch t.msg.Event.Type { + case Upload: + uploadEvent, ok := t.msg.Event.Content.(*UploadEvent) + if !ok { + log.Error(errors.New("failed to convert interface{} to *UploadEvent")) + return + } + err = cli.handleUploadEvent(uploadEvent) + default: + err = errors.New("EventMessage type unexpected") + } + if err != nil { + cli.log.Error("error occurred in Client.call", log.Error(err)) + } + if t.cb != nil { + t.cb(t.msg, err) + } +} + +// upload upload object to service(BOS, CEPH or AWS S3) +func (cli *Client) upload(f, remotePath string, meta map[string]string) error { + fsize, md5 := cli.fileSizeMd5(f) + saved, err := cli.checkFile(remotePath, md5) + if err != nil { + return errors.Trace(err) + } + if saved { + return nil + } + if cli.cfg.Limit.Enable { + month := time.Unix(0, time.Now().UnixNano()).Format("2006-01") + err := cli.checkData(fsize, month) + if err != nil { + atomic.AddUint64(&cli.fs.limit, 1) + return errors.Errorf("failed to pass data check: %s", err.Error()) + } + err = cli.putObjectWithStats(cli.cfg.Bucket, remotePath, f, meta) + if err != nil { + return err + } + return cli.increaseData(fsize, month) + } + err = cli.putObjectWithStats(cli.cfg.Bucket, remotePath, f, meta) + if err != nil { + return errors.Trace(err) + } + return nil +} -// Client interface -type Client interface { - CallAsync(msg *EventMessage, cb func(msg *EventMessage, err error)) error - Start() error - io.Closer +func (cli *Client) putObjectWithStats(bucket, remotePath, f string, meta map[string]string) error { + err := cli.handler.PutObjectFromFile(bucket, remotePath, f, meta) + if err != nil { + cli.log.Error("failed to put object from file", log.Any("localFile", f), log.Any("remotePath", remotePath), log.Any("bucket", bucket), log.Error(err)) + atomic.AddUint64(&cli.fs.fail, 1) + return errors.Trace(err) + } + cli.log.Info("put object from file successfully", log.Any("localFile", f), log.Any("remotePath", remotePath), log.Any("bucket", bucket)) + atomic.AddUint64(&cli.fs.success, 1) + return nil } -// NewClient can create a ruler -func NewClient(cfg ClientInfo, r report) (Client, error) { - return NewStorageClient(cfg, r) +func (cli *Client) handleUploadEvent(e *UploadEvent) error { + if strings.Contains(e.LocalPath, "..") { + return errors.Errorf("failed to pass LocalPath (%s) check: the local path can't contains ..", e.LocalPath) + } + var t string + p, err := filepath.EvalSymlinks(path.Join(cli.pwd, e.LocalPath)) + if err != nil { + atomic.AddUint64(&cli.fs.deleted, 1) + return errors.Errorf("failed get real dir path: %s", err.Error()) + } + if ok := utils.FileExists(p); ok { + if e.Zip { + t = path.Join(cli.cfg.TempPath, uuid.Generate().String()) + cli.arch = &archiver.Zip{ + CompressionLevel: flate.DefaultCompression, + MkdirAll: true, + SelectiveCompression: true, + OverwriteExisting: true, + } + } else { + t = p + } + } else if ok = utils.DirExists(p); ok { + t = path.Join(cli.cfg.TempPath, uuid.Generate().String()) + if e.Zip { + cli.arch = &archiver.Zip{ + CompressionLevel: flate.DefaultCompression, + MkdirAll: true, + SelectiveCompression: true, + OverwriteExisting: true, + } + } else { + cli.arch = &archiver.Tar{ + MkdirAll: true, + OverwriteExisting: true, + } + } + } else { + atomic.AddUint64(&cli.fs.deleted, 1) + return errors.Errorf("failed to find path: %s", p) + } + if t != p { + defer os.RemoveAll(t) + } + + if cli.arch != nil { + if err := cli.arch.Archive([]string{p}, t); err != nil { + return errors.Errorf("failed to zip/tar dir: %s", err.Error()) + } + } + + return cli.upload(t, e.RemotePath, e.Meta) +} + +func (cli *Client) checkFile(remotePath, md5 string) (bool, error) { + res, err := cli.handler.FileExists(cli.cfg.Bucket, remotePath, md5) + if err != nil { + return false, errors.Trace(err) + } + return res, nil +} + +func (cli *Client) fileSizeMd5(f string) (int64, string) { + fi, err := os.Stat(f) + if err != nil { + cli.log.Error("failed to get file info: %s", log.Error(err)) + return 0, "" + } + fsize := fi.Size() + + md5, err := utils.CalculateFileMD5(f) + if err != nil { + cli.log.Error("failed to calculate file MD5", log.Any("filename", f), log.Error(err)) + return fsize, "" + } + return fsize, md5 +} + +func (cli *Client) checkData(fsize int64, month string) error { + if cli.cfg.Limit.Data <= 0 { + return errors.New("limit data should be greater than 0(Byte)") + } + cli.lock.Lock() + defer cli.lock.Unlock() + if _, ok := cli.stats.Months[month]; ok { + newSize := cli.stats.Months[month].Bytes + fsize + if newSize > cli.cfg.Limit.Data { + return errors.New("exceeds max upload data size of this month，stop to upload and will reset next month") + } + } + return nil +} + +func (cli *Client) increaseData(fsize int64, month string) error { + cli.lock.Lock() + defer cli.lock.Unlock() + if _, ok := cli.stats.Months[month]; !ok { + cli.stats.Months[month] = &Item{} + } + cli.stats.Total.Bytes = cli.stats.Total.Bytes + fsize + cli.stats.Total.Count++ + cli.stats.Months[month].Bytes = cli.stats.Months[month].Bytes + fsize + cli.stats.Months[month].Count++ + return DumpYAML(cli.cfg.Limit.Path, &cli.stats) +} + +func (cli *Client) recording() error { + defer cli.log.Debug("client recording task stopped") + t := time.NewTicker(cli.cfg.Record.Interval) + defer t.Stop() + for { + select { + case <-cli.tomb.Dying(): + return nil + case <-t.C: + cli.log.Info("client stats data", + log.Any("success", cli.fs.success), + log.Any("fail", cli.fs.fail), + log.Any("limit", cli.fs.limit), + log.Any("deleted", cli.fs.deleted)) + } + } +} + +// Close close client and all worker +func (cli *Client) Close() error { + cli.log.Info("client starts to close") + defer cli.log.Info("client closed") + + cli.pool.Release() + cli.tomb.Kill(nil) + return cli.tomb.Wait() } diff --git a/baetyl-remote-object/client_test.go b/baetyl-remote-object/client_test.go index c81cf48..28917e9 100644 --- a/baetyl-remote-object/client_test.go +++ b/baetyl-remote-object/client_test.go @@ -1,49 +1,372 @@ package main import ( + "io/ioutil" + "os" + "path" "testing" "time" "github.com/stretchr/testify/assert" ) -const createClientError = "failed to create storage client (test): kind type unexpected" +func newClient() (*Client, error) { + cfg.Kind = "S3" + cfg.Region = "us-east-1" + storageClient, err := NewClient(*cfg) + return storageClient, err +} + +func generateTempPath(prefix string) (string, error) { + dir, err := ioutil.TempDir("", prefix) + if err != nil { + return "", err + } + tmpfile, err := ioutil.TempFile(dir, prefix) + if err != nil { + return "", err + } + fpath := tmpfile.Name() + ".yml" + return fpath, nil +} + +func TestNewStorageClient(t *testing.T) { + // round 1: report is not nil + _, err := newClient() + assert.Equal(t, err.Error(), "failed to make dir (): mkdir : no such file or directory") +} + +// CallAsync and invoke +func TestCallAsync(t *testing.T) { + cfg.Kind = "S3" + cfg.Region = "us-east-1" + + tempPath, err := generateTempPath("example") + defer os.RemoveAll(path.Dir(tempPath)) + assert.Nil(t, err) + + cfg.TempPath = tempPath + + statsPath, err := generateTempPath("test") + defer os.RemoveAll(path.Dir(statsPath)) + assert.Nil(t, err) + + cfg.Limit.Path = statsPath + cfg.Pool.Worker = 10 + cfg.Pool.Idletime = time.Duration(30000000000) + + // create storage client + storageClient, err := newClient() + assert.Nil(t, err) + defer storageClient.Close() + + msg := &EventMessage{ + ID: 1, + QOS: uint32(1), + Topic: "t", + Event: &Event{ + Time: time.Now(), + Type: EventType("UPLOAD"), + Content: nil, + }, + } + err = storageClient.CallAsync(msg, mockRuleHook) + assert.Nil(t, err) +} + +func mockRuleHook(msg *EventMessage, err error) { + return +} + +func TestCall(t *testing.T) { + cfg.Kind = "S3" + cfg.Region = "us-east-1" + + tempPath, err := generateTempPath("example") + defer os.RemoveAll(path.Dir(tempPath)) + assert.Nil(t, err) + + cfg.TempPath = tempPath + + statsPath, err := generateTempPath("test") + defer os.RemoveAll(path.Dir(statsPath)) + assert.Nil(t, err) + + cfg.Limit.Path = statsPath + cfg.Pool.Worker = 10 + cfg.Pool.Idletime = time.Duration(30000000000) + + // create storage client + storageClient, err := newClient() + assert.Nil(t, err) + defer storageClient.Close() + + task1 := &Task{ + msg: &EventMessage{ + ID: 1, + QOS: uint32(1), + Topic: "t", + Event: &Event{ + Time: time.Now(), + Type: EventType("TEST"), // unsupported event type + Content: nil, + }, + }, + cb: mockRuleHook, + } + // start call + storageClient.call(task1) + + // unsupported struct when convert to Task struct + task2 := map[string]string{} + storageClient.call(task2) +} + +func TestUpload(t *testing.T) { + cfg.Kind = "S3" + cfg.Region = "us-east-1" + cfg.Bucket = "default" + + tempPath, err := generateTempPath("example") + defer os.RemoveAll(path.Dir(tempPath)) + assert.Nil(t, err) + + cfg.TempPath = tempPath + + statsPath, err := generateTempPath("test") + defer os.RemoveAll(path.Dir(statsPath)) + assert.Nil(t, err) + + cfg.Limit.Path = statsPath + cfg.Pool.Worker = 10 + cfg.Pool.Idletime = time.Duration(30000000000) + + // create storage client + storageClient, err := newClient() + assert.Nil(t, err) + defer storageClient.Close() + + // round 1: local file is not exist + err = storageClient.upload("var/test/file", "default", map[string]string{}) + assert.NotNil(t, err) + assert.Equal(t, "EmptyStaticCreds: static credentials are empty", err.Error()) + + // round 2: file exists without limit data + storageClient.cfg.Bucket = "Bucket" + storageClient.cfg.MultiPart.PartSize = 1048576000 + storageClient.cfg.MultiPart.Concurrency = 10 + err = storageClient.upload("./example/etc/baetyl/service-bos.yml", "var/file/service.yml", map[string]string{}) + assert.NotNil(t, err) + assert.Equal(t, "EmptyStaticCreds: static credentials are empty", err.Error()) // without AccessKey and SecretKey + + // round 3: file exists with limit data + storageClient.cfg.Limit.Enable = true + storageClient.cfg.Limit.Data = 1073741824 + storageClient.cfg.Limit.Path = "var/lib/baetyl/data/stats.yml" + storageClient.stats.Months = map[string]*Item{ + "2019-09": &Item{ + Bytes: 21234345, + Count: 20, + }} + err = storageClient.upload("./example/test/baetyl/service.yml", "var/file/service.yml", map[string]string{}) + assert.NotNil(t, err) + assert.Equal(t, "EmptyStaticCreds: static credentials are empty", err.Error()) // without AccessKey and SecretKey +} + +func TestHandleUploadEvent(t *testing.T) { + cfg.Kind = "S3" + cfg.Region = "us-east-1" + + tempPath, err := generateTempPath("example") + defer os.RemoveAll(path.Dir(tempPath)) + assert.Nil(t, err) + + cfg.TempPath = tempPath + + statsPath, err := generateTempPath("test") + defer os.RemoveAll(path.Dir(statsPath)) + assert.Nil(t, err) + + cfg.Limit.Path = statsPath + cfg.Pool.Worker = 10 + cfg.Pool.Idletime = time.Duration(30000000000) + + // create storage client + storageClient, err := newClient() + assert.Nil(t, err) + defer storageClient.Close() + + // contains '..' of local path + e := &UploadEvent{ + RemotePath: "var/file/service.yml", + LocalPath: "./example/etc/baetyl/service-bos.yml", + Zip: false, + Meta: make(map[string]string), + } + + // wrong path + e.LocalPath = "../example/etc/baetyl/service-bos.yml" + err = storageClient.handleUploadEvent(e) + assert.Error(t, err) + + // real path + e.LocalPath = "./example/etc/baetyl/service-bos.yml" + storageClient.cfg.Bucket = "Bucket" + storageClient.cfg.MultiPart.PartSize = 1048576000 + storageClient.cfg.MultiPart.Concurrency = 10 + err = storageClient.handleUploadEvent(e) + assert.NotNil(t, err) + assert.Equal(t, "EmptyStaticCreds: static credentials are empty", err.Error()) // without AccessKey and SecretKey -func example(map[string]interface{}) error { - return nil + // zip is true, upload file + e.Zip = true + e.RemotePath = "var/file/test.zip" + err = storageClient.handleUploadEvent(e) + assert.NotNil(t, err) + assert.Equal(t, "failed to zip/tar dir: checking extension: filename must have a .zip extension", err.Error()) + + // zip is true, upload directory + e.LocalPath = "./example" + err = storageClient.handleUploadEvent(e) + assert.NotNil(t, err) + assert.Equal(t, "failed to zip/tar dir: checking extension: filename must have a .zip extension", err.Error()) + + // zip is false, tar directory and upload + e.Zip = false + e.RemotePath = "var/file/test.tar" + err = storageClient.handleUploadEvent(e) + assert.NotNil(t, err) + assert.Equal(t, "failed to zip/tar dir: checking extension: filename must have a .tar extension", err.Error()) } -var r = report(example) +func TestCheckFile(t *testing.T) { + cfg.Kind = "S3" + cfg.Region = "us-east-1" + cfg.Bucket = "default" + + tempPath, err := generateTempPath("example") + defer os.RemoveAll(path.Dir(tempPath)) + assert.Nil(t, err) + + cfg.TempPath = tempPath + + statsPath, err := generateTempPath("test") + defer os.RemoveAll(path.Dir(statsPath)) + assert.Nil(t, err) + + cfg.Limit.Path = statsPath + cfg.Pool.Worker = 10 + cfg.Pool.Idletime = time.Duration(30000000000) + + // create storage client + storageClient, err := newClient() + assert.Nil(t, err) + defer storageClient.Close() + + remotePath := "var/file/service.yml" + md5 := "4a0fb0ea68b05a84234e420d1f8cb32b" -var cfg = &ClientInfo{ - Name: "test", - Report: struct { - Interval time.Duration `yaml:"interval" json:"interval" default:"1m"` - }{ - Interval: time.Duration(1000000000), - }, + rlt, err := storageClient.checkFile(remotePath, md5) + assert.Equal(t, err.Error(), "EmptyStaticCreds: static credentials are empty") + assert.Equal(t, false, rlt) } -func TestNewClient(t *testing.T) { - // round 1: test BOS client - cfg.Kind = Kind("BOS") - _, err := NewClient(*cfg, r) +func TestCheckData(t *testing.T) { + cfg.Kind = "S3" + cfg.Region = "us-east-1" + + tempPath, err := generateTempPath("example") + defer os.RemoveAll(path.Dir(tempPath)) + assert.Nil(t, err) + + cfg.TempPath = tempPath + + statsPath, err := generateTempPath("test") + defer os.RemoveAll(path.Dir(statsPath)) assert.Nil(t, err) - // round 2: test CEPH client - cfg.Kind = Kind("CEPH") - _, err = NewClient(*cfg, r) + cfg.Pool.Worker = 10 + cfg.Pool.Idletime = time.Duration(30000000000) + + // round 1: limit data less than 0(Byte) + cfg.Limit.Enable = true + cfg.Limit.Data = -1 + cfg.Limit.Path = "var/lib/baetyl/data/stats.yml" + + // create storage client + storageClient, err := newClient() + assert.Nil(t, err) + defer storageClient.Close() + + err = storageClient.checkData(200, "2019-09") + assert.NotNil(t, err) + assert.Equal(t, "limit data should be greater than 0(Byte)", err.Error()) + + // round 2: limit data greater than 0(Byte), will exceed + storageClient.cfg.Limit.Data = 1073741824 + storageClient.stats.Months = map[string]*Item{ + "2019-09": &Item{ + Bytes: 221212425, + Count: 20, + }} + err = storageClient.checkData(2000000000, "2019-09") + assert.NotNil(t, err) + assert.Equal(t, "exceeds max upload data size of this month，stop to upload and will reset next month", err.Error()) + + // round 3: limit data greater than 0(Byte), will not exceed + storageClient.cfg.Limit.Data = 107374182400 + err = storageClient.checkData(2000000000, "2019-09") assert.Nil(t, err) +} - // round 3: test AWS S3 client - cfg.Kind = Kind("S3") +func TestStartAndClose(t *testing.T) { + cfg.Kind = "S3" cfg.Region = "us-east-1" - _, err = NewClient(*cfg, r) + + cfg.TempPath = "/usr/data.yml" + defer os.RemoveAll("var/") + + statsPath, err := generateTempPath("test") + defer os.RemoveAll(path.Dir(statsPath)) assert.Nil(t, err) - // round 4: test default - cfg.Kind = Kind("TEST") - _, err = NewClient(*cfg, r) + cfg.Limit.Path = statsPath + cfg.Pool.Worker = 10 + cfg.Pool.Idletime = time.Duration(30000000000) + + // create storage client + _, err = newClient() assert.NotNil(t, err) - assert.Equal(t, createClientError, err.Error()) + assert.Contains(t, err.Error(), "failed to make dir (/usr/data.yml)") + + // cannot make directory of limit path + dir, err := ioutil.TempDir("", "example") + assert.Nil(t, err) + defer os.RemoveAll(dir) + cfg.TempPath = dir + cfg.Limit.Path = "/var/file/service.yml" + // create storage client + _, err = newClient() + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "failed to make dir (/var/file)") + + // invalid size for pool + cfg.Pool.Worker = 0 + //cfg.Pool.Idletime = time.Duration(30000000000) + tmpfile, err := ioutil.TempFile(dir, "test") + cfg.Limit.Path = tmpfile.Name() + ".yml" + // create storage client + _, err = newClient() + assert.NotNil(t, err) + assert.Equal(t, "failed to create a pool: invalid size for pool", err.Error()) + + // storage client start successfully + cfg.Pool.Worker = 10 + cfg.Pool.Idletime = time.Duration(30000000000) + // create storage client + storageClient, err := newClient() + assert.NoError(t, err) + + err = storageClient.Close() + assert.NoError(t, err) } diff --git a/baetyl-remote-object/config.go b/baetyl-remote-object/config.go index 4e50c21..f3e86be 100644 --- a/baetyl-remote-object/config.go +++ b/baetyl-remote-object/config.go @@ -4,7 +4,7 @@ import ( "io/ioutil" "time" - "github.com/baetyl/baetyl/protocol/mqtt" + "github.com/baetyl/baetyl-go/v2/errors" "github.com/docker/go-units" yaml "gopkg.in/yaml.v2" ) @@ -14,15 +14,51 @@ type Kind string // The type of event from cloud const ( - Bos Kind = "BOS" - Ceph Kind = "CEPH" - S3 Kind = "S3" + Bos Kind = "BOS" + S3 Kind = "S3" ) -// Config config of module +// Config config of rule type Config struct { - Clients []ClientInfo `yaml:"clients" json:"clients" default:"[]"` - Rules []Rule `yaml:"rules" json:"rules" default:"[]"` + Clients []ClientInfo `yaml:"clients" json:"clients"` + Rules []RuleInfo `yaml:"rules" json:"rules"` +} + +// ClientInfo client config +type ClientInfo struct { + ObjectConfig `yaml:",inline" json:",inline"` + Name string `yaml:"name" json:"name" validate:"nonzero"` + Kind Kind `yaml:"kind" json:"kind" validate:"nonzero"` + TempPath string `yaml:"temppath" json:"temppath" default:"var/lib/baetyl/tmp"` + Timeout time.Duration `yaml:"timeout" json:"timeout" default:"30s"` + Backoff Backoff `yaml:"backoff" json:"backoff"` + Pool Pool `yaml:"pool" json:"pool"` + MultiPart MultiPart `yaml:"multipart" json:"multipart"` + Limit Limit `yaml:"limit" json:"limit"` + Record struct { + Interval time.Duration `yaml:"interval" json:"interval" default:"1m"` + } `yaml:"record" json:"record"` +} + +// ObjectConfig (BOS、AWSS3) client config +type ObjectConfig struct { + Endpoint string `yaml:"endpoint" json:"endpoint"` + Region string `yaml:"region" json:"region" default:"us-east-1"` + Ak string `yaml:"ak" json:"ak" validate:"nonzero"` + Sk string `yaml:"sk" json:"sk" validate:"nonzero"` + Bucket string `yaml:"bucket" json:"bucket" validate:"nonzero"` +} + +// RuleInfo rule info +type RuleInfo struct { + Name string `yaml:"name" json:"name" validate:"nonzero"` + Source struct { + QOS uint32 `yaml:"qos" json:"qos" validate:"min=0, max=1"` + Topic string `yaml:"topic" json:"topic" validate:"nonzero"` + } `yaml:"source" json:"source" validate:"nonzero"` + Target struct { + Client string `yaml:"client" json:"client" default:"baetyl-broker"` + } `yaml:"target" json:"target"` } // Backoff policy @@ -38,37 +74,6 @@ type Pool struct { Idletime time.Duration `yaml:"idletime" json:"idletime" default:"30s"` // delay time } -// ClientInfo client config -type ClientInfo struct { - Name string `yaml:"name" json:"name" validate:"nonzero"` - Address string `yaml:"address" json:"address"` - Region string `yaml:"region" json:"region" default:"us-east-1"` - Ak string `yaml:"ak" json:"ak" validate:"nonzero"` - Sk string `yaml:"sk" json:"sk" validate:"nonzero"` - Kind Kind `yaml:"kind" json:"kind" validate:"nonzero"` - Timeout time.Duration `yaml:"timeout" json:"timeout" default:"30s"` - Backoff Backoff `yaml:"backoff" json:"backoff"` - Pool Pool `yaml:"pool" json:"pool"` - Bucket string `yaml:"bucket" json:"bucket" validate:"nonzero"` - TempPath string `yaml:"temppath" json:"temppath" default:"var/lib/baetyl/tmp"` - MultiPart MultiPart `yaml:"multipart" json:"multipart"` - Limit Limit `yaml:"limit" json:"limit"` - Report struct { - Interval time.Duration `yaml:"interval" json:"interval" default:"1m"` - } `yaml:"report" json:"report"` -} - -// Rule function rule config -type Rule struct { - Hub struct { - ClientID string `yaml:"clientid" json:"clientid"` - Subscriptions []mqtt.TopicInfo `yaml:"subscriptions" json:"subscriptions" default:"[]"` - } `yaml:"hub" json:"hub"` - Client struct { - Name string `yaml:"name" json:"name" validate:"nonzero"` - } `yaml:"client" json:"client"` -} - // MultiPart config type MultiPart struct { PartSize int64 `yaml:"partsize" json:"partsize" default:"1048576000"` @@ -150,11 +155,11 @@ type Stats struct { func DumpYAML(path string, in interface{}) error { bytes, err := yaml.Marshal(in) if err != nil { - return err + return errors.Trace(err) } err = ioutil.WriteFile(path, bytes, 0755) if err != nil { - return err + return errors.Trace(err) } return nil } diff --git a/baetyl-remote-object/config_test.go b/baetyl-remote-object/config_test.go index 4d65ced..b74b19f 100644 --- a/baetyl-remote-object/config_test.go +++ b/baetyl-remote-object/config_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/baetyl/baetyl/utils" + "github.com/baetyl/baetyl-go/v2/utils" "github.com/stretchr/testify/assert" ) @@ -18,8 +18,8 @@ func TestConfig(t *testing.T) { assert.Nil(t, err) assert.Len(t, c.Clients, 1) - assert.Equal(t, "baidu-bos", c.Clients[0].Name) - assert.Equal(t, "bj.bcebos.com", c.Clients[0].Address) + assert.Equal(t, "baidubos", c.Clients[0].Name) + assert.Equal(t, "bj.bcebos.com", c.Clients[0].Endpoint) assert.Equal(t, Kind("BOS"), c.Clients[0].Kind) assert.Equal(t, int64(10485760), c.Clients[0].MultiPart.PartSize) assert.Equal(t, 10, c.Clients[0].MultiPart.Concurrency) @@ -30,11 +30,12 @@ func TestConfig(t *testing.T) { assert.Equal(t, true, c.Clients[0].Limit.Enable) assert.Equal(t, int64(9663676416), c.Clients[0].Limit.Data) assert.Equal(t, "var/lib/baetyl/data/stats.yml", c.Clients[0].Limit.Path) - assert.Equal(t, time.Duration(60000000000), c.Clients[0].Report.Interval) + assert.Equal(t, time.Duration(60000000000), c.Clients[0].Record.Interval) assert.Len(t, c.Rules, 1) - assert.Equal(t, "remote-write-bos", c.Rules[0].Hub.ClientID) - assert.Equal(t, "t", c.Rules[0].Hub.Subscriptions[0].Topic) + assert.Equal(t, uint32(1), c.Rules[0].Source.QOS) + assert.Equal(t, "broker/topic1", c.Rules[0].Source.Topic) + assert.Equal(t, "baidubos", c.Rules[0].Target.Client) // round 2: load bad configuration yaml file err = utils.LoadYAML("example/test/baetyl/service.yml", &c) diff --git a/baetyl-remote-object/example/etc/baetyl/service-bos.yml b/baetyl-remote-object/example/etc/baetyl/service-bos.yml index 2d88e2b..dabfa6f 100644 --- a/baetyl-remote-object/example/etc/baetyl/service-bos.yml +++ b/baetyl-remote-object/example/etc/baetyl/service-bos.yml @@ -1,13 +1,9 @@ -hub: - address: tcp://127.0.0.1:1883 - username: test - password: hahaha clients: - - name: baidu-bos - address: bj.bcebos.com + - name: baidubos # 名称 + kind: BOS # mqtt 类型 + endpoint: bj.bcebos.com # 地址 ak: XXXXXXXXXXXXXXXXXXXXXXXXXX sk: XXXXXXXXXXXXXXXXXXXXXXXXXX - kind: BOS multipart: partsize: 10m concurrency: 10 @@ -21,14 +17,15 @@ clients: enable: true data: 9g path: var/lib/baetyl/data/stats.yml + rules: - - hub: - clientid: remote-write-bos - subscriptions: - # hub topic - - topic: t - client: - name: baidu-bos + - name: rule1 # 规则名称，必须保持唯一 + source: # 消息源 + topic: broker/topic1 # 消息主题 + qos: 1 # 消息质量 + target: # 消息目的地 + client: baidubos # object 目标节点 + logger: path: var/log/baetyl/service.log level: "debug" \ No newline at end of file diff --git a/baetyl-remote-object/example/etc/baetyl/service-ceph.yml b/baetyl-remote-object/example/etc/baetyl/service-ceph.yml deleted file mode 100644 index 1bbfa41..0000000 --- a/baetyl-remote-object/example/etc/baetyl/service-ceph.yml +++ /dev/null @@ -1,34 +0,0 @@ -hub: - address: tcp://127.0.0.1:1883 - username: test - password: hahaha -clients: - - name: baidu-bos - address: http://127.0.0.1:8080 - ak: XXXXXXXXXXXXXXXXXXXXXXXXXX - sk: XXXXXXXXXXXXXXXXXXXXXXXXXX - kind: CEPH - multipart: - partsize: 10m - concurrency: 10 - pool: - worker: 1000 - idletime: 30s - bucket: bos-remote-demo - temppath: var/lib/baetyl/tmp - # max 1000g - limit: - enable: true - data: 9g - path: var/lib/baetyl/data/stats.yml -rules: - - hub: - clientid: remote-write-bos - subscriptions: - # hub topic - - topic: t - client: - name: baidu-bos -logger: - path: var/log/baetyl/service.log - level: "debug" \ No newline at end of file diff --git a/baetyl-remote-object/example/etc/baetyl/service-minio.yml b/baetyl-remote-object/example/etc/baetyl/service-minio.yml new file mode 100644 index 0000000..1774664 --- /dev/null +++ b/baetyl-remote-object/example/etc/baetyl/service-minio.yml @@ -0,0 +1,31 @@ +clients: + - name: minio + kind: S3 + endpoint: http://127.0.0.1:8080 + ak: XXXXXXXXXXXXXXXXXXXXXXXX + sk: XXXXXXXXXXXXXXXXXXXXXXXX + multipart: + partsize: 10m + concurrency: 10 + pool: + worker: 1000 + idletime: 30s + bucket: bos-remote-demo + temppath: var/lib/baetyl/tmp + # max 1000g + limit: + enable: true + data: 9g + path: var/lib/baetyl/data/stats.yml + +rules: + - name: rule3 + source: + topic: broker/topic3 + qos: 1 + target: + client: minio + +logger: + path: var/log/baetyl/service.log + level: "debug" \ No newline at end of file diff --git a/baetyl-remote-object/example/etc/baetyl/service-s3.yml b/baetyl-remote-object/example/etc/baetyl/service-s3.yml index 097580d..54cffdd 100644 --- a/baetyl-remote-object/example/etc/baetyl/service-s3.yml +++ b/baetyl-remote-object/example/etc/baetyl/service-s3.yml @@ -1,13 +1,9 @@ -var/lib: - address: tcp://127.0.0.1:1883 - username: test - password: hahaha clients: - - name: baidu-bos + - name: awss3 + kind: S3 region: us-east-2 ak: XXXXXXXXXXXXXXXXXXXXXXXX sk: XXXXXXXXXXXXXXXXXXXXXXXX - kind: S3 multipart: partsize: 10m concurrency: 10 @@ -21,14 +17,15 @@ clients: enable: true data: 9g path: var/lib/baetyl/data/stats.yml + rules: - - hub: - clientid: remote-write-bos - subscriptions: - # hub topic - - topic: t - client: - name: baidu-bos + - name: rule2 + source: + topic: broker/topic2 + qos: 1 + target: + client: awss3 + logger: path: var/log/baetyl/service.log level: "debug" \ No newline at end of file diff --git a/baetyl-remote-object/main.go b/baetyl-remote-object/main.go index 6225835..a6e185c 100644 --- a/baetyl-remote-object/main.go +++ b/baetyl-remote-object/main.go @@ -1,59 +1,45 @@ package main import ( - "fmt" - - baetyl "github.com/baetyl/baetyl/sdk/baetyl-go" + "github.com/baetyl/baetyl-go/v2/context" ) -// mo bridge module of mqtt servers -type mo struct { - cfg Config - rrs []*Ruler -} - func main() { - baetyl.Run(func(ctx baetyl.Context) error { + context.Run(func(ctx context.Context) error { var cfg Config - err := ctx.LoadConfig(&cfg) + err := ctx.LoadCustomConfig(&cfg) if err != nil { return err } + // clients - clients := make(map[string]Client) + clients := make(map[string]*Client) + defer func() { + for _, c := range clients { + c.Close() + } + }() for _, c := range cfg.Clients { - clients[c.Name], err = NewClient(c, ctx.ReportInstance) - defer clients[c.Name].Close() + client, err := NewClient(c) if err != nil { return err } + clients[c.Name] = client } + // rulers rulers := make([]*Ruler, 0) - for _, rule := range cfg.Rules { - cli, ok := clients[rule.Client.Name] - if !ok { - return fmt.Errorf("client (%s) not found", rule.Client.Name) - } - ruler := NewRuler(rule, ctx.Config().Hub, cli) - rulers = append(rulers, ruler) - } defer func() { - for _, ruler := range rulers { - ruler.Close() + for _, r := range rulers { + r.Close() } }() - for _, cli := range clients { - err := cli.Start() - if err != nil { - return err - } - } - for _, ruler := range rulers { - err := ruler.Start() + for _, ruleInfo := range cfg.Rules { + ruler, err := NewRuler(ruleInfo, clients, ctx.ServiceName()) if err != nil { return err } + rulers = append(rulers, ruler) } ctx.Wait() return nil diff --git a/baetyl-remote-object/package.yml b/baetyl-remote-object/package.yml deleted file mode 100644 index 4e19f97..0000000 --- a/baetyl-remote-object/package.yml +++ /dev/null @@ -1,4 +0,0 @@ -# Code generated by github.com/baetyl/baetyl/sdk/baetyl-go/generate.go. DO NOT EDIT. -# source: github.com/baetyl/baetyl/sdk/baetyl-go/templates/package-go.yml - -entry: bin/baetyl-remote-object diff --git a/baetyl-remote-object/program.yml b/baetyl-remote-object/program.yml new file mode 100644 index 0000000..7cf2eeb --- /dev/null +++ b/baetyl-remote-object/program.yml @@ -0,0 +1 @@ +entry: "baetyl-remote-object" diff --git a/baetyl-remote-object/rule_test.go b/baetyl-remote-object/rule_test.go index 132fe91..e29cd08 100644 --- a/baetyl-remote-object/rule_test.go +++ b/baetyl-remote-object/rule_test.go @@ -2,71 +2,34 @@ package main import ( "testing" + "time" - "github.com/256dpi/gomqtt/packet" - "github.com/baetyl/baetyl/protocol/mqtt" "github.com/stretchr/testify/assert" ) -var ru = &Rule{ - Hub: struct { - ClientID string `yaml:"clientid" json:"clientid"` - Subscriptions []mqtt.TopicInfo `yaml:"subscriptions" json:"subscriptions" default:"[]"` - }{ - ClientID: "", - Subscriptions: []mqtt.TopicInfo(nil), - }, - Client: struct { - Name string `yaml:"name" json:"name" validate:"nonzero"` - }{ - Name: "example", - }, -} - -func TestDefaults(t *testing.T) { - // round 1: hub client ID is empty - hub := new(mqtt.ClientInfo) - defaults(ru, hub) - assert.Equal(t, []mqtt.TopicInfo(nil), hub.Subscriptions) - assert.Equal(t, "example", hub.ClientID) - - // round 2: hub client ID is not empty - ru.Hub.ClientID = "hub-test-1" - ru.Hub.Subscriptions = []mqtt.TopicInfo{mqtt.TopicInfo{Topic: "t"}} - defaults(ru, hub) - assert.Equal(t, ru.Hub.Subscriptions, hub.Subscriptions) - assert.Equal(t, "hub-test-1", hub.ClientID) -} +func TestRule(t *testing.T) { + clients := map[string]*Client{ + "cli1": &Client{}, + } -func TestProcessEvent(t *testing.T) { - pkt := &packet.Publish{ - ID: packet.ID(1), - Message: packet.Message{ - Topic: "t", - QOS: 0, - Payload: []byte("for test"), + ruleInfo := RuleInfo{ + Name: "", + Source: struct { + QOS uint32 `yaml:"qos" json:"qos" validate:"min=0, max=1"` + Topic string `yaml:"topic" json:"topic" validate:"nonzero"` + }{ + QOS: 1, + Topic: "t1", + }, + Target: struct { + Client string `yaml:"client" json:"client" default:"baetyl-broker"` + }{ + Client: "cli1", }, } - hub := new(mqtt.ClientInfo) - cfg.Kind = Kind("BOS") - cli, err := NewClient(*cfg, r) - assert.Nil(t, err) - ruler := NewRuler(*ru, *hub, cli) - event, err := ruler.processEvent(pkt) - assert.Nil(t, event) - assert.NotNil(t, err) - assert.Equal(t, "event invalid: event type unexpected", err.Error()) - pkt.Message.Payload = []byte( - `{ - "type": "UPLOAD", - "content": { - "remotePath": "image/test.png", - "zip": false, - "localPath": "var/lib/baetyl/image/test.png" - }}`) - event, err = ruler.processEvent(pkt) - assert.Nil(t, err) - assert.Equal(t, EventType("UPLOAD"), event.Type) - assert.Equal(t, &UploadEvent{RemotePath: "image/test.png", LocalPath: "var/lib/baetyl/image/test.png", Zip: false}, event.Content) + ruler, err := NewRuler(ruleInfo, clients, "ruletest") + assert.NoError(t, err) + time.Sleep(time.Second) + ruler.Close() } diff --git a/baetyl-remote-object/ruler.go b/baetyl-remote-object/ruler.go index 8ebcf3e..7556892 100644 --- a/baetyl-remote-object/ruler.go +++ b/baetyl-remote-object/ruler.go @@ -5,76 +5,80 @@ import ( "sync" "github.com/256dpi/gomqtt/packet" - "github.com/baetyl/baetyl/logger" - "github.com/baetyl/baetyl/protocol/mqtt" + "github.com/baetyl/baetyl-go/v2/errors" + "github.com/baetyl/baetyl-go/v2/log" + "github.com/baetyl/baetyl-go/v2/mqtt" ) // Ruler struct type Ruler struct { - rule *Rule - cli Client - hub *mqtt.Dispatcher - log logger.Logger - tm sync.Map + sourceCli *mqtt.Client + targetCli *Client + log *log.Logger + tm sync.Map } // NewRuler can create a ruler -func NewRuler(rule Rule, hub mqtt.ClientInfo, cli Client) *Ruler { - defaults(&rule, &hub) - log := logger.WithField("rule", rule.Client.Name) - return &Ruler{ - rule: &rule, - hub: mqtt.NewDispatcher(hub, log), - cli: cli, - log: log, +func NewRuler(rule RuleInfo, targets map[string]*Client, serviceName string) (*Ruler, error) { + targetCli, ok := targets[rule.Target.Client] + if !ok { + return nil, errors.Errorf("client (%s) not found", rule.Target.Client) } -} -// Start can create a ruler -func (r *Ruler) Start() error { - return r.hub.Start(r) + mqttCli := getBrokerClient(rule.Source.QOS, rule.Source.Topic, fmt.Sprintf("%s-rule-%s", serviceName, rule.Name)) + ruler := &Ruler{ + sourceCli: mqttCli, + targetCli: targetCli, + log: log.With(log.Any("rule", rule.Name)), + } + err := mqttCli.Start(mqtt.NewObserverWrapper(func(pkt *packet.Publish) error { + event, err := ruler.processEvent(pkt) + if err != nil { + ruler.log.Error("error occurred in ruler.processEvent", log.Error(err)) + return nil + } + msg := &EventMessage{ + ID: uint64(pkt.ID), + QOS: uint32(pkt.Message.QOS), + Topic: pkt.Message.Topic, + Event: event, + } + err = ruler.RuleHandler(msg) + if err != nil { + ruler.log.Error("error occurred in ruler.RuleHandler", log.Error(err)) + } + return nil + }, func(*packet.Puback) error { + return nil + }, func(err error) { + ruler.log.Error("error occurs in source", log.Error(err)) + })) + if err != nil { + ruler.log.Error("error occurred when mqtt client start", log.Error(err)) + } + return ruler, nil } // Close can create a ruler func (r *Ruler) Close() { - r.hub.Close() -} + r.log.Info("rule starts to close") + defer r.log.Info("rule closed") -// ProcessPublish can create a ruler -func (r *Ruler) ProcessPublish(pkt *packet.Publish) error { - event, err := r.processEvent(pkt) - if err != nil { - r.log.Errorf(err.Error()) - return err + // sourceCli is internal client + if r.sourceCli != nil { + r.sourceCli.Close() } - msg := &EventMessage{ - ID: uint64(pkt.ID), - QOS: uint32(pkt.Message.QOS), - Topic: pkt.Message.Topic, - Event: event, - } - return r.RuleHandler(msg) } func (r *Ruler) processEvent(pkt *packet.Publish) (*Event, error) { - r.log.Debugln("event: ", string(pkt.Message.Payload)) + r.log.Debug("ruler received a event: ", log.Any("payload", string(pkt.Message.Payload))) e, err := NewEvent(pkt.Message.Payload) if err != nil { - return nil, fmt.Errorf("event invalid: %s", err.Error()) + return nil, errors.Errorf("event invalid: %s", err.Error()) } return e, nil } -// ProcessPuback test -func (r *Ruler) ProcessPuback(pkt *packet.Puback) error { - return nil -} - -// ProcessError test -func (r *Ruler) ProcessError(err error) { - r.log.Errorf(err.Error()) -} - // RuleHandler filter topic & handler func (r *Ruler) RuleHandler(msg *EventMessage) error { if msg.QOS == 1 { @@ -84,7 +88,7 @@ func (r *Ruler) RuleHandler(msg *EventMessage) error { return nil } } - return r.cli.CallAsync(msg, r.callback) + return r.targetCli.CallAsync(msg, r.callback) } func (r *Ruler) callback(msg *EventMessage, err error) { @@ -92,20 +96,28 @@ func (r *Ruler) callback(msg *EventMessage, err error) { if err == nil { puback := packet.NewPuback() puback.ID = packet.ID(msg.ID) - r.hub.Send(puback) + err := r.sourceCli.Send(puback) + if err != nil { + r.log.Error("failed to send mqtt msg", log.Error(err)) + } } r.tm.Delete(msg.ID) } if err != nil { - r.log.Errorf(err.Error()) + r.log.Error("failed to invoke object client", log.Error(err)) } } -func defaults(rule *Rule, hub *mqtt.ClientInfo) { - if rule.Hub.ClientID != "" { - hub.ClientID = rule.Hub.ClientID - } else { - hub.ClientID = rule.Client.Name +func getBrokerClient(qos uint32, topic, clientID string) *mqtt.Client { + mqttCfg := mqtt.NewClientOptions() + mqttCfg.ClientID = clientID + mqttCfg.Address = "tcp://baetyl-broker:1883" + mqttCfg.CleanSession = false + mqttCfg.Subscriptions = []mqtt.Subscription{ + { + QOS: mqtt.QOS(qos), + Topic: topic, + }, } - hub.Subscriptions = rule.Hub.Subscriptions + return mqtt.NewClient(mqttCfg) } diff --git a/baetyl-remote-object/storage_client.go b/baetyl-remote-object/storage_client.go deleted file mode 100644 index 0e6a193..0000000 --- a/baetyl-remote-object/storage_client.go +++ /dev/null @@ -1,336 +0,0 @@ -package main - -import ( - "compress/flate" - "fmt" - "os" - "path" - "path/filepath" - "strings" - "sync" - "sync/atomic" - "time" - - util "github.com/baetyl/baetyl-go/utils" - "github.com/baetyl/baetyl/logger" - "github.com/docker/distribution/uuid" - "github.com/mholt/archiver" - "github.com/panjf2000/ants" -) - -type arch interface { - // Archive archivies an archive file on disk - Archive(source []string, destination string) error -} - -// nonArch origin file -type nonArch struct{} - -// Archive no action -func (a *nonArch) Archive(source []string, destination string) error { - return nil -} - -// Task StorageClient -type Task struct { - msg *EventMessage - cb func(msg *EventMessage, err error) -} - -// FileStats upload stats -type FileStats struct { - success uint64 - fail uint64 - limit uint64 - deleted uint64 -} - -// StorageClient StorageClient -type StorageClient struct { - cfg ClientInfo - sh IObjectStorage - stats Stats - pwd string - fs *FileStats - report report - arch arch - log logger.Logger - tomb util.Tomb - pool *ants.PoolWithFunc - lock sync.RWMutex -} - -// NewStorageClient creates a new newStorageClient -func NewStorageClient(cfg ClientInfo, r report) (*StorageClient, error) { - pwd, err := os.Getwd() - if err != nil { - return nil, err - } - sh, err := NewObjectStorageHandler(cfg) - if err != nil { - return nil, fmt.Errorf("failed to create storage client (%s): %s", cfg.Name, err.Error()) - } - b := &StorageClient{ - cfg: cfg, - sh: sh, - pwd: pwd, - report: r, - arch: &nonArch{}, - fs: &FileStats{}, - log: logger.WithField("storage client", cfg.Name), - } - if r != nil { - return b, b.tomb.Go(b.reporting) - } - return b, nil -} - -// CallAsync submit task -func (cli *StorageClient) CallAsync(msg *EventMessage, cb func(msg *EventMessage, err error)) error { - if !cli.tomb.Alive() { - return fmt.Errorf("client (%s) closed", cli.cfg.Name) - } - return cli.invoke(msg, cb) -} - -func (cli *StorageClient) invoke(msg *EventMessage, cb func(msg *EventMessage, err error)) error { - if cli.pool.Running() == cli.cfg.Pool.Worker { - cb(msg, fmt.Errorf("failed to submit task: no worker can be used")) - return nil - } - task := &Task{ - msg: msg, - cb: cb, - } - if err := cli.pool.Invoke(task); err != nil { - cb(msg, fmt.Errorf("failed to invoke pool task: %s", err.Error())) - return nil - } - return nil -} - -func (cli *StorageClient) call(task interface{}) { - t, ok := task.(*Task) - if !ok { - return - } - var err error - switch t.msg.Event.Type { - case Upload: - uploadEvent := t.msg.Event.Content.(*UploadEvent) - err = cli.handleUploadEvent(uploadEvent) - default: - err = fmt.Errorf("EventMessage type unexpected") - } - if err != nil { - cli.log.Errorf("failed to fetch: %s", err.Error()) - } - if t.cb != nil { - t.cb(t.msg, err) - } -} - -// upload upload object to service(BOS, CEPH or AWS S3) -func (cli *StorageClient) upload(f, remotePath string, meta map[string]string) error { - fsize, md5 := cli.fileSizeMd5(f) - saved := cli.checkFile(remotePath, md5) - if saved { - return nil - } - if cli.cfg.Limit.Enable { - month := time.Unix(0, time.Now().UnixNano()).Format("2006-01") - err := cli.checkData(fsize, month) - if err != nil { - cli.log.Errorf("failed to pass data check: %s", err.Error()) - atomic.AddUint64(&cli.fs.limit, 1) - return nil - } - err = cli.putObjectWithStats(cli.cfg.Bucket, remotePath, f, meta) - if err != nil { - return err - } - return cli.increaseData(fsize, month) - } - err := cli.putObjectWithStats(cli.cfg.Bucket, remotePath, f, meta) - if err != nil { - return err - } - return nil -} - -func (cli *StorageClient) putObjectWithStats(bucket, remotePath, f string, meta map[string]string) error { - err := cli.sh.PutObjectFromFile(bucket, remotePath, f, meta) - if err != nil { - atomic.AddUint64(&cli.fs.fail, 1) - return err - } - atomic.AddUint64(&cli.fs.success, 1) - return nil -} - -func (cli *StorageClient) handleUploadEvent(e *UploadEvent) error { - if strings.Contains(e.LocalPath, "..") { - cli.log.Errorf("failed to pass LocalPath (%s) check: the local path can't contains ..", e.LocalPath) - return nil - } - var t string - p, err := filepath.EvalSymlinks(path.Join(cli.pwd, e.LocalPath)) - if err != nil { - cli.log.Errorf("failed get real dir path: %s", err.Error()) - atomic.AddUint64(&cli.fs.deleted, 1) - return nil - } - if ok := util.FileExists(p); ok { - if e.Zip { - t = path.Join(cli.cfg.TempPath, uuid.Generate().String()) - cli.arch = &archiver.Zip{ - CompressionLevel: flate.DefaultCompression, - MkdirAll: true, - SelectiveCompression: true, - OverwriteExisting: true, - } - } else { - t = p - } - } else if ok = util.DirExists(p); ok { - t = path.Join(cli.cfg.TempPath, uuid.Generate().String()) - if e.Zip { - cli.arch = &archiver.Zip{ - CompressionLevel: flate.DefaultCompression, - MkdirAll: true, - SelectiveCompression: true, - OverwriteExisting: true, - } - } else { - cli.arch = &archiver.Tar{ - MkdirAll: true, - OverwriteExisting: true, - } - } - } else { - atomic.AddUint64(&cli.fs.deleted, 1) - return fmt.Errorf("failed to find path: %s", p) - } - err = cli.arch.Archive([]string{p}, t) - if t != p { - defer os.RemoveAll(t) - } - if err != nil { - return fmt.Errorf("failed to zip/tar dir: %s", err.Error()) - } - return cli.upload(t, e.RemotePath, e.Meta) -} - -func (cli *StorageClient) checkFile(remotePath, md5 string) bool { - return cli.sh.FileExists(cli.cfg.Bucket, remotePath, md5) -} - -func (cli *StorageClient) fileSizeMd5(f string) (int64, string) { - fi, err := os.Stat(f) - if err != nil { - cli.log.Errorf("failed to get file info: %s", err.Error()) - return 0, "" - } - fsize := fi.Size() - md5, err := util.CalculateFileMD5(f) - if err != nil { - cli.log.Errorf("failed to calculate file (%s) MD5: %s", f, err.Error()) - return fsize, "" - } - return fsize, md5 -} - -func (cli *StorageClient) checkData(fsize int64, month string) error { - if cli.cfg.Limit.Data <= 0 { - return fmt.Errorf("limit data should be greater than 0(Byte)") - } - cli.lock.RLock() - defer cli.lock.RUnlock() - if _, ok := cli.stats.Months[month]; ok { - new := cli.stats.Months[month].Bytes + fsize - if new > cli.cfg.Limit.Data { - return fmt.Errorf("exceeds max upload data size of this month，stop to upload and will reset next month") - } - } - return nil -} - -func (cli *StorageClient) increaseData(fsize int64, month string) error { - cli.lock.Lock() - defer cli.lock.Unlock() - if _, ok := cli.stats.Months[month]; !ok { - cli.stats.Months[month] = &Item{} - } - cli.stats.Total.Bytes = cli.stats.Total.Bytes + fsize - cli.stats.Total.Count++ - cli.stats.Months[month].Bytes = cli.stats.Months[month].Bytes + fsize - cli.stats.Months[month].Count++ - return DumpYAML(cli.cfg.Limit.Path, &cli.stats) -} - -func (cli *StorageClient) reporting() error { - defer cli.log.Debugf("storage client reporting task stopped") - var err error - t := time.NewTicker(cli.cfg.Report.Interval) - defer t.Stop() - for { - select { - case <-cli.tomb.Dying(): - return nil - case <-t.C: - stats := map[string]interface{}{ - cli.cfg.Name: map[string]interface{}{ - "success": cli.fs.success, - "fail": cli.fs.fail, - "limit": cli.fs.limit, - "deleted": cli.fs.deleted, - }, - } - err = cli.report(stats) - if err != nil { - cli.log.Warnf("failed to report storage client file stats") - } - cli.log.Debugln(stats) - } - } -} - -// Start start all worker -func (cli *StorageClient) Start() error { - err := os.MkdirAll(cli.cfg.TempPath, 0755) - if err != nil { - cli.log.Errorf("failed to make dir (%s): %s", cli.cfg.TempPath, err.Error()) - return err - } - if ok := util.FileExists(cli.cfg.Limit.Path); !ok { - basepath := path.Dir(cli.cfg.Limit.Path) - err = os.MkdirAll(basepath, 0755) - if err != nil { - cli.log.Errorf("failed to make dir (%s): %s", basepath, err.Error()) - return err - } - f, err := os.Create(cli.cfg.Limit.Path) - defer f.Close() - if err != nil { - cli.log.Errorf("failed to make file (%s): %s", cli.cfg.Limit.Path, err.Error()) - return err - } - } - util.LoadYAML(cli.cfg.Limit.Path, &cli.stats) - p, err := ants.NewPoolWithFunc(cli.cfg.Pool.Worker, cli.call, ants.WithExpiryDuration(cli.cfg.Pool.Idletime)) - if err != nil { - cli.log.Errorf("failed to create a pool: %s", err.Error()) - return err - } - cli.pool = p - cli.log.Debugf("storage client start") - return nil -} - -// Close close client and all worker -func (cli *StorageClient) Close() error { - cli.pool.Release() - cli.tomb.Kill(nil) - cli.log.Debugf("storage client closed") - return cli.tomb.Wait() -} diff --git a/baetyl-remote-object/storage_client_test.go b/baetyl-remote-object/storage_client_test.go deleted file mode 100644 index 9df48d6..0000000 --- a/baetyl-remote-object/storage_client_test.go +++ /dev/null @@ -1,278 +0,0 @@ -package main - -import ( - "io/ioutil" - "os" - "path" - "testing" - "time" - - "github.com/baetyl/baetyl/protocol/mqtt" - "github.com/stretchr/testify/assert" -) - -func newStorageClient(r report) (*StorageClient, error) { - cfg.Kind = Kind("S3") - cfg.Region = "us-east-1" - storageClient, err := NewStorageClient(*cfg, r) - return storageClient, err -} - -func generateTempPath(prefix string) (string, error) { - dir, err := ioutil.TempDir("", prefix) - if err != nil { - return "", err - } - tmpfile, err := ioutil.TempFile(dir, prefix) - if err != nil { - return "", err - } - fpath := tmpfile.Name() + ".yml" - return fpath, nil -} - -func TestNewStorageClient(t *testing.T) { - // round 1: report is not nil - storageClient, err := newStorageClient(r) - assert.Nil(t, err) - assert.Equal(t, Kind("S3"), storageClient.cfg.Kind) - assert.Equal(t, "test", storageClient.cfg.Name) - - // round 2: report is nil - storageClient, err = newStorageClient(nil) - assert.Nil(t, err) - assert.Equal(t, Kind("S3"), storageClient.cfg.Kind) - assert.Equal(t, "test", storageClient.cfg.Name) -} - -// CallAsync and invoke -func TestCallAsync(t *testing.T) { - // create storage client - storageClient, err := newStorageClient(r) - assert.Nil(t, err) - - // start storage client - tempPath, err := generateTempPath("example") - defer os.RemoveAll(path.Dir(tempPath)) - assert.Nil(t, err) - storageClient.cfg.TempPath = tempPath - statsPath, err := generateTempPath("test") - defer os.RemoveAll(path.Dir(statsPath)) - assert.Nil(t, err) - storageClient.cfg.Limit.Path = statsPath - storageClient.cfg.Pool.Worker = 10 - storageClient.cfg.Pool.Idletime = time.Duration(30000000000) - err = storageClient.Start() - assert.Nil(t, err) - defer storageClient.Close() - - msg := &EventMessage{ - ID: 1, - QOS: uint32(1), - Topic: "t", - Event: &Event{ - Time: time.Now(), - Type: EventType("UPLOAD"), - Content: nil, - }, - } - hub := new(mqtt.ClientInfo) - ruler := NewRuler(*ru, *hub, Client(storageClient)) - err = storageClient.CallAsync(msg, ruler.callback) - assert.Nil(t, err) -} - -func TestCall(t *testing.T) { - // create storage client - storageClient, err := newStorageClient(r) - assert.Nil(t, err) - // create ruler - hub := new(mqtt.ClientInfo) - ruler := NewRuler(*ru, *hub, Client(storageClient)) - task1 := &Task{ - msg: &EventMessage{ - ID: 1, - QOS: uint32(1), - Topic: "t", - Event: &Event{ - Time: time.Now(), - Type: EventType("TEST"), // unsupported event type - Content: nil, - }, - }, - cb: ruler.callback, - } - // start call - storageClient.call(task1) - - // unsupported struct when convert to Task struct - task2 := map[string]string{} - storageClient.call(task2) -} - -func TestUpload(t *testing.T) { - // create storage client - storageClient, err := newStorageClient(r) - assert.Nil(t, err) - - // round 1: local file is not exist - err = storageClient.upload("var/test/file", "", map[string]string{}) - assert.NotNil(t, err) - assert.Equal(t, "open var/test/file: no such file or directory", err.Error()) - - // round 2: file exists without limit data - storageClient.cfg.Bucket = "Bucket" - storageClient.cfg.MultiPart.PartSize = 1048576000 - storageClient.cfg.MultiPart.Concurrency = 10 - err = storageClient.upload("./example/test/baetyl/service.yml", "var/file/service.yml", map[string]string{}) - assert.NotNil(t, err) - assert.Equal(t, "EmptyStaticCreds: static credentials are empty", err.Error()) // without AccessKey and SecretKey - - // round 3: file exists with limit data - storageClient.cfg.Limit.Enable = true - storageClient.cfg.Limit.Data = 1073741824 - storageClient.cfg.Limit.Path = "var/lib/baetyl/data/stats.yml" - storageClient.stats.Months = map[string]*Item{ - "2019-09": &Item{ - Bytes: 21234345, - Count: 20, - }} - err = storageClient.upload("./example/test/baetyl/service.yml", "var/file/service.yml", map[string]string{}) - assert.NotNil(t, err) - assert.Equal(t, "EmptyStaticCreds: static credentials are empty", err.Error()) // without AccessKey and SecretKey -} - -func TestHandleUploadEvent(t *testing.T) { - // create storage client - storageClient, err := newStorageClient(r) - assert.Nil(t, err) - - // contains '..' of local path - e := &UploadEvent{ - RemotePath: "var/file/service.yml", - LocalPath: "../example/test/baetyl/service.yml", - Zip: false, - Meta: make(map[string]string), - } - err = storageClient.handleUploadEvent(e) - assert.Nil(t, err) - - // wrong path - e.LocalPath = "./test/baetyl/service.yml" - err = storageClient.handleUploadEvent(e) - assert.Nil(t, err) - - // real path - e.LocalPath = "./example/test/baetyl/service.yml" - storageClient.cfg.Bucket = "Bucket" - storageClient.cfg.MultiPart.PartSize = 1048576000 - storageClient.cfg.MultiPart.Concurrency = 10 - err = storageClient.handleUploadEvent(e) - assert.NotNil(t, err) - assert.Equal(t, "EmptyStaticCreds: static credentials are empty", err.Error()) // without AccessKey and SecretKey - - // zip is true, upload file - e.Zip = true - e.RemotePath = "var/file/test.zip" - err = storageClient.handleUploadEvent(e) - assert.NotNil(t, err) - assert.Equal(t, "failed to zip/tar dir: checking extension: filename must have a .zip extension", err.Error()) - - // zip is true, upload directory - e.LocalPath = "./example" - err = storageClient.handleUploadEvent(e) - assert.NotNil(t, err) - assert.Equal(t, "failed to zip/tar dir: checking extension: filename must have a .zip extension", err.Error()) - - // zip is false, tar directory and upload - e.Zip = false - e.RemotePath = "var/file/test.tar" - err = storageClient.handleUploadEvent(e) - assert.NotNil(t, err) - assert.Equal(t, "failed to zip/tar dir: checking extension: filename must have a .tar extension", err.Error()) -} - -func TestCheckFile(t *testing.T) { - remotePath := "var/file/service.yml" - md5 := "4a0fb0ea68b05a84234e420d1f8cb32b" - storageClient, err := newStorageClient(r) - assert.Nil(t, err) - rlt := storageClient.checkFile(remotePath, md5) - assert.Equal(t, false, rlt) -} - -func TestCheckData(t *testing.T) { - // create storage client - storageClient, err := newStorageClient(r) - assert.Nil(t, err) - - // round 1: limit data less than 0(Byte) - storageClient.cfg.Limit.Enable = true - storageClient.cfg.Limit.Data = -1 - storageClient.cfg.Limit.Path = "var/lib/baetyl/data/stats.yml" - err = storageClient.checkData(200, "2019-09") - assert.NotNil(t, err) - assert.Equal(t, "limit data should be greater than 0(Byte)", err.Error()) - - // round 2: limit data greater than 0(Byte), will exceed - storageClient.cfg.Limit.Data = 1073741824 - storageClient.stats.Months = map[string]*Item{ - "2019-09": &Item{ - Bytes: 221212425, - Count: 20, - }} - err = storageClient.checkData(2000000000, "2019-09") - assert.NotNil(t, err) - assert.Equal(t, "exceeds max upload data size of this month，stop to upload and will reset next month", err.Error()) - - // round 3: limit data greater than 0(Byte), will not exceed - storageClient.cfg.Limit.Data = 107374182400 - err = storageClient.checkData(2000000000, "2019-09") - assert.Nil(t, err) -} - -func TestStartAndClose(t *testing.T) { - // create storage client - storageClient, err := newStorageClient(r) - assert.Nil(t, err) - - // file is not exist - storageClient.cfg.TempPath = "var/file/test" - defer os.RemoveAll("var/") - err = storageClient.Start() - assert.NotNil(t, err) - assert.Equal(t, "open : no such file or directory", err.Error()) - - // cannot make directory of temppath - storageClient.cfg.TempPath = "/usr/data.yml" - err = storageClient.Start() - assert.NotNil(t, err) - assert.Equal(t, "mkdir /usr/data.yml: operation not permitted", err.Error()) - - // cannot make directory of limit path - dir, err := ioutil.TempDir("", "example") - assert.Nil(t, err) - defer os.RemoveAll(dir) - storageClient.cfg.TempPath = dir - storageClient.cfg.Limit.Path = "/var/file/service.yml" - err = storageClient.Start() - assert.NotNil(t, err) - assert.Equal(t, "mkdir /var/file: permission denied", err.Error()) - - // invalid size for pool - tmpfile, err := ioutil.TempFile(dir, "test") - storageClient.cfg.Limit.Path = tmpfile.Name() + ".yml" - err = storageClient.Start() - assert.NotNil(t, err) - assert.Equal(t, "invalid size for pool", err.Error()) - - // storage client start successfully - storageClient.cfg.Pool.Worker = 10 - storageClient.cfg.Pool.Idletime = time.Duration(30000000000) - err = storageClient.Start() - assert.Nil(t, err) - - // storage client close - err = storageClient.Close() - assert.Nil(t, err) -} diff --git a/baetyl-remote-object/storage_handler.go b/baetyl-remote-object/storage_handler.go index a5d9588..7eaec21 100644 --- a/baetyl-remote-object/storage_handler.go +++ b/baetyl-remote-object/storage_handler.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/baetyl/baetyl-go/v2/errors" "github.com/baidubce/bce-sdk-go/bce" "github.com/baidubce/bce-sdk-go/services/bos" "github.com/baidubce/bce-sdk-go/services/bos/api" @@ -20,19 +21,17 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3manager" ) -// IObjectStorage interface -type IObjectStorage interface { +// ObjectStorage interface +type StorageHandler interface { PutObjectFromFile(Bucket, remotePath, filename string, meta map[string]string) error - FileExists(Bucket, remotePath, md5 string) bool + FileExists(Bucket, remotePath, md5 string) (bool, error) } // NewObjectStorageHandler NewObjectStorageHandler -func NewObjectStorageHandler(cfg ClientInfo) (IObjectStorage, error) { +func NewObjectStorageHandler(cfg ClientInfo) (StorageHandler, error) { switch cfg.Kind { case Bos: return NewBosHandler(cfg) - case Ceph: - return NewCephClient(cfg) case S3: return NewS3Client(cfg) default: @@ -47,17 +46,17 @@ type BosHandler struct { } // NewBosHandler creates a new newBosClient -func NewBosHandler(cfg ClientInfo) (*BosHandler, error) { - bos, err := bos.NewClient(cfg.Ak, cfg.Sk, cfg.Address) +func NewBosHandler(cfg ClientInfo) (StorageHandler, error) { + cli, err := bos.NewClient(cfg.Ak, cfg.Sk, cfg.Endpoint) if err != nil { - return nil, fmt.Errorf("failed to create bos client (%s): %s", cfg.Name, err.Error()) + return nil, errors.Errorf("failed to create bos client (%s): %s", cfg.Name, err.Error()) } - bos.MultipartSize = cfg.MultiPart.PartSize - bos.MaxParallel = (int64)(cfg.MultiPart.Concurrency) - bos.Config.ConnectionTimeoutInMillis = (int)(cfg.Timeout / time.Millisecond) - bos.Config.Retry = bce.NewBackOffRetryPolicy(cfg.Backoff.Max, (int64)(cfg.Backoff.Delay/time.Millisecond), (int64)(cfg.Backoff.Base/time.Millisecond)) + cli.MultipartSize = cfg.MultiPart.PartSize + cli.MaxParallel = (int64)(cfg.MultiPart.Concurrency) + cli.Config.ConnectionTimeoutInMillis = (int)(cfg.Timeout / time.Millisecond) + cli.Config.Retry = bce.NewBackOffRetryPolicy(cfg.Backoff.Max, (int64)(cfg.Backoff.Delay/time.Millisecond), (int64)(cfg.Backoff.Base/time.Millisecond)) b := &BosHandler{ - bos: bos, + bos: cli, cfg: cfg, } return b, nil @@ -68,18 +67,19 @@ func (cli *BosHandler) PutObjectFromFile(Bucket, remotePath, filename string, me args := new(api.PutObjectArgs) args.UserMeta = meta _, err := cli.bos.PutObjectFromFile(Bucket, remotePath, filename, args) - return err + return errors.Trace(err) } // FileExists FileExists -func (cli *BosHandler) FileExists(Bucket, remotePath, md5 string) bool { - res, _ := cli.bos.GetObjectMeta(Bucket, remotePath) - if res != nil { - if res.ObjectMeta.ContentMD5 == md5 { - return true - } +func (cli *BosHandler) FileExists(Bucket, remotePath, md5 string) (bool, error) { + res, err := cli.bos.GetObjectMeta(Bucket, remotePath) + if err != nil { + return false, errors.Trace(err) + } + if res.ObjectMeta.ContentMD5 == md5 { + return true, nil } - return false + return false, nil } // S3Handler S3Handler @@ -89,43 +89,24 @@ type S3Handler struct { cfg ClientInfo } -// NewCephClient creates a new NewCephClient -func NewCephClient(cfg ClientInfo) (*S3Handler, error) { - // Configure to use S3 Server +// NewS3Client creates a new NewS3Client +func NewS3Client(cfg ClientInfo) (StorageHandler, error) { s3Config := &aws.Config{ Credentials: credentials.NewStaticCredentials(cfg.Ak, cfg.Sk, ""), - Endpoint: aws.String(cfg.Address), - Region: aws.String("us-east-1"), - DisableSSL: aws.Bool(!strings.HasPrefix(cfg.Address, "https")), + Endpoint: aws.String(cfg.Endpoint), + Region: aws.String(cfg.Region), + DisableSSL: aws.Bool(!strings.HasPrefix(cfg.Endpoint, "https")), S3ForcePathStyle: aws.Bool(true), } - newSession := session.New(s3Config) - s3Client := s3.New(newSession) - uploader := s3manager.NewUploader(newSession) - c := &S3Handler{ - s3Client: s3Client, - cfg: cfg, - uploader: uploader, - } - return c, nil -} - -// NewS3Client creates a new NewS3Client -func NewS3Client(cfg ClientInfo) (*S3Handler, error) { - // Configure to use S3 Server - s3Config := &aws.Config{ - Credentials: credentials.NewStaticCredentials(cfg.Ak, cfg.Sk, ""), - Region: aws.String(cfg.Region), + sessionProvider, err := session.NewSession(s3Config) + if err != nil { + return nil, errors.Trace(err) } - newSession := session.New(s3Config) - s3Client := s3.New(newSession) - uploader := s3manager.NewUploader(newSession) - c := &S3Handler{ - s3Client: s3Client, + return &S3Handler{ + s3Client: s3.New(sessionProvider), cfg: cfg, - uploader: uploader, - } - return c, nil + uploader: s3manager.NewUploader(sessionProvider), + }, nil } // PutObjectFromFile upload file @@ -135,10 +116,10 @@ func (cli *S3Handler) PutObjectFromFile(Bucket, remotePath, filename string, met Metadata[k] = &v } f, err := os.Open(filename) - defer f.Close() if err != nil { - return err + return errors.Trace(err) } + defer f.Close() params := &s3manager.UploadInput{ Bucket: aws.String(Bucket), // Required Key: aws.String(remotePath), // Required @@ -147,30 +128,28 @@ func (cli *S3Handler) PutObjectFromFile(Bucket, remotePath, filename string, met } ctx, cancel := context.WithTimeout(context.Background(), cli.cfg.Timeout) defer cancel() - // _, err = cli.ceph.PutObjectWithContext(ctx, params) _, err = cli.uploader.UploadWithContext(ctx, params, func(u *s3manager.Uploader) { u.PartSize = cli.cfg.MultiPart.PartSize u.LeavePartsOnError = true u.Concurrency = cli.cfg.MultiPart.Concurrency }) //并发数 - - return err + return errors.Trace(err) } // FileExists FileExists -func (cli *S3Handler) FileExists(Bucket, remotePath, md5 string) bool { +func (cli *S3Handler) FileExists(Bucket, remotePath, md5 string) (bool, error) { cparams := &s3.HeadObjectInput{ Bucket: aws.String(Bucket), Key: aws.String(remotePath), } ho, err := cli.s3Client.HeadObject(cparams) if err != nil { - return false + return false, errors.Trace(err) } input, _ := hex.DecodeString(strings.Replace(*ho.ETag, "\"", "", -1)) encodeString := base64.StdEncoding.EncodeToString(input) - if encodeString != md5 { - return false + if encodeString == md5 { + return true, nil } - return true + return false, nil } diff --git a/baetyl-remote-object/storage_handler_test.go b/baetyl-remote-object/storage_handler_test.go index 60dab4d..1ebda8f 100644 --- a/baetyl-remote-object/storage_handler_test.go +++ b/baetyl-remote-object/storage_handler_test.go @@ -2,16 +2,26 @@ package main import ( "testing" + "time" "github.com/aws/aws-sdk-go/awstesting/mock" "github.com/aws/aws-sdk-go/awstesting/unit" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/baetyl/baetyl/utils" + "github.com/baetyl/baetyl-go/v2/utils" "github.com/docker/distribution/uuid" "github.com/stretchr/testify/assert" ) +var cfg = &ClientInfo{ + Name: "test", + Record: struct { + Interval time.Duration `yaml:"interval" json:"interval" default:"1m"` + }{ + Interval: time.Duration(1000000000), + }, +} + func TestNewBosHandler(t *testing.T) { // var bosHandler *BosHandler // round 1: create bos handler normally with none empty AccessKey and SecretKey @@ -42,7 +52,7 @@ func TestNewBosHandler(t *testing.T) { } func TestPutObjectFromFile(t *testing.T) { - cfg.Kind = Kind("S3") + cfg.Kind = "S3" cfg.Region = "us-east-2" cfg.MultiPart.PartSize = 1048576000 cfg.MultiPart.Concurrency = 10 @@ -59,7 +69,6 @@ func TestPutObjectFromFile(t *testing.T) { } func TestFileExists(t *testing.T) { - cfg.Kind = Kind("S3") sc := mock.NewMockClient() u := s3manager.NewUploader(unit.Session) s3Handler := &S3Handler{ @@ -67,8 +76,8 @@ func TestFileExists(t *testing.T) { uploader: u, cfg: *cfg, } - md5, err := utils.CalculateFileMD5("example/test/baetyl/service.yml") + md5, err := utils.CalculateFileMD5("example/etc/baetyl/service-bos.yml") assert.Nil(t, err) - rlt := s3Handler.FileExists("Bucket", "var/file/service.yml", md5) - assert.Equal(t, false, rlt) + _, err = s3Handler.FileExists("Bucket", "var/file/service.yml", md5) + assert.Equal(t, err.Error(), "MissingRegion: could not find region configuration") } diff --git a/go.mod b/go.mod index 69f68aa..9756e12 100644 --- a/go.mod +++ b/go.mod @@ -1,23 +1,17 @@ module github.com/baetyl/baetyl-remote -replace ( - github.com/256dpi/gomqtt => github.com/256dpi/gomqtt v0.12.2 - github.com/docker/docker => github.com/docker/engine v0.0.0-20191007211215-3e077fc8667a - github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.1-0.20190307181833-2b18fe1d885e -) - go 1.13 require ( - github.com/256dpi/gomqtt v0.13.0 - github.com/aws/aws-sdk-go v1.25.36 - github.com/baetyl/baetyl v0.0.0-20200117072245-2f3abdcf12ee - github.com/baetyl/baetyl-go v0.1.5 + github.com/256dpi/gomqtt v0.14.2 + github.com/aws/aws-sdk-go v1.32.8 + github.com/baetyl/baetyl-go/v2 v2.0.61 github.com/baidubce/bce-sdk-go v0.9.5 github.com/docker/distribution v2.7.1+incompatible github.com/docker/go-units v0.4.0 github.com/mholt/archiver v3.1.1+incompatible github.com/panjf2000/ants v1.3.0 - github.com/stretchr/testify v1.4.0 - gopkg.in/yaml.v2 v2.2.7 + github.com/stretchr/testify v1.5.1 + golang.org/x/tools v0.0.0-20191205225056-3393d29bb9fe // indirect + gopkg.in/yaml.v2 v2.2.8 ) diff --git a/go.sum b/go.sum index 396ae20..234d078 100644 --- a/go.sum +++ b/go.sum @@ -1,170 +1,176 @@ +cloud.google.com/go v0.16.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/256dpi/gomqtt v0.12.2 h1:/heAnll2UrBesa9x1wgSaQxJMD+IVlPo9PnUsMP31S8= -github.com/256dpi/gomqtt v0.12.2/go.mod h1:umNEh2nPBiUyN5uNnmkX6Tsf0Kx8/BWaJuNCIj1J+ks= -github.com/256dpi/gomqtt v0.13.0 h1:kpIBQ8oMyY3Wg+MqIpiH9Y2TTcjhw9Ed+CcoueOLiNk= -github.com/256dpi/gomqtt v0.13.0/go.mod h1:vWiB7Vt8R/9Jx9WAD6YDvBN3SKzWMRbdOcLf0cbPUkI= -github.com/256dpi/mercury v0.1.0 h1:4SqjOOOJUhi/TWGH277qi6TPJOPxp5NXm+NO0Rfvii8= -github.com/256dpi/mercury v0.1.0/go.mod h1:W2/eVt6tqfSn5J8en63oGNmnZSb66PUo0e5YBzSHkkU= +github.com/256dpi/gomqtt v0.14.2 h1:TBUKjqBJtU8tfZwiRbpZevzvMWUlzZlbhuzL0hIghok= +github.com/256dpi/gomqtt v0.14.2/go.mod h1:2y4qpvxskFCnQpLIdEY5HG55zqXgAjB88xqrBbZsQRg= github.com/256dpi/mercury v0.2.0 h1:ImB0JYuZ28kwp2MpqnMdQFSD3z9mgaNYHrSjYuyP0LI= github.com/256dpi/mercury v0.2.0/go.mod h1:xxgxZSQO7VUwxGLpk8yRVe/WF0MKH7nCIwSh4kUVMy4= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/aws/aws-sdk-go v1.25.36 h1:4+TL/Y2G5hsR1zdfHmjNG1ou1WEqsSWk8v7m1GaDKyo= -github.com/aws/aws-sdk-go v1.25.36/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/baetyl/baetyl v0.0.0-20200117072245-2f3abdcf12ee h1:6UHbjvV4dWwjK/auQEd2qmlGVfaxdTPl3sO87ckyqq8= -github.com/baetyl/baetyl v0.0.0-20200117072245-2f3abdcf12ee/go.mod h1:rGRTDtRz5qaS10KxfbBtIL9QF4AN3wliBd5+qpjVDKY= -github.com/baetyl/baetyl-go v0.0.7/go.mod h1:bGk+qk5pIiMJcADfbRw91CbcIgvUWdN/3gqp8yH5usw= -github.com/baetyl/baetyl-go v0.1.5 h1:PQ6/6L6W9rrw+pHYuFMyQwUmdpLXo24aTjRuS8NiVTg= -github.com/baetyl/baetyl-go v0.1.5/go.mod h1:bGk+qk5pIiMJcADfbRw91CbcIgvUWdN/3gqp8yH5usw= +github.com/aws/aws-sdk-go v1.32.8 h1:ULxiQqR1eZ+k2/1gqv3GYAjkunlS7ncVU2eL801t08M= +github.com/aws/aws-sdk-go v1.32.8/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/baetyl/baetyl-go/v2 v2.0.61 h1:nbGaFqFSD7LRg19f+6YI1q92wvDlCrlQ9hZ94GS1l2c= +github.com/baetyl/baetyl-go/v2 v2.0.61/go.mod h1:ETX1SbGqT1I4miZoCVb7wwa6HCH639xat8j+4lVabtQ= github.com/baidubce/bce-sdk-go v0.9.5 h1:x07/Mp0DZa6xzuOXM2ESMF4eWC2WarQYBRs08HuguKw= github.com/baidubce/bce-sdk-go v0.9.5/go.mod h1:T3yEA2H7hXAlvniSEJRsPlDYlh8OEZZzH0zlIP/1JIY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/containerd/containerd v1.3.0 h1:xjvXQWABwS2uiv3TWgQt5Uth60Gu86LTGZXMJkjc7rY= -github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/creasty/defaults v1.3.0 h1:uG+RAxYbJgOPCOdKEcec9ZJXeva7Y6mj/8egdzwmLtw= -github.com/creasty/defaults v1.3.0/go.mod h1:CIEEvs7oIVZm30R8VxtFJs+4k201gReYyuYHJxZc68I= +github.com/containerd/containerd v1.3.4 h1:3o0smo5SKY7H6AJCmJhsnCjR2/V2T8VmiHt7seN2/kI= +github.com/containerd/containerd v1.3.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/creasty/defaults v1.4.0 h1:Pz90duUjIzkmCznPtRSpamL+ET00QOxyA+kIgpRDp/E= +github.com/creasty/defaults v1.4.0/go.mod h1:9UWnPlI41ASz+YJswP5aK5S79d6QH60/Ioz52OXV9X8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/engine v0.0.0-20191007211215-3e077fc8667a/go.mod h1:3CPr2caMgTHxxIAZgEMd3uLYPDlRvPqCpyeRf6ncPcY= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= -github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= +github.com/frankban/quicktest v1.10.0 h1:Gfh+GAJZOAoKZsIZeZbdn2JF10kN1XHNvjsvQK8gVkE= +github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= +github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= -github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/go-ozzo/ozzo-routing v2.1.4+incompatible h1:gQmNyAwMnBHr53Nma2gPTfVVc6i2BuAwCWPam2hIvKI= +github.com/go-ozzo/ozzo-routing v2.1.4+incompatible/go.mod h1:hvoxy5M9SJaY0viZvcCsODidtUm5CzRbYKEWuQpr+2A= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.6.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/gddo v0.0.0-20200611223618-a4829ef13274 h1:q1WDRWSuDPX5UBTPq+QYr6WPOgnz4Hb5k+gY00SdJZg= +github.com/golang/gddo v0.0.0-20200611223618-a4829ef13274/go.mod h1:sam69Hju0uq+5uvLJUMDlsKlQ21Vrs1Kd/1YFPNYdOU= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/lint v0.0.0-20170918230701-e5d664eb928e/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.1.1-0.20171103154506-982329095285/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/websocket v1.3.0 h1:r/LXc0VJIMd0rCMsc6DxgczaQtoCwCLatnfXmSYcXx8= -github.com/gorilla/websocket v1.3.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jolestar/go-commons-pool v2.0.0+incompatible/go.mod h1:ChJYIbIch0DMCSU6VU0t0xhPoWDR2mMFIQek3XWU0s8= +github.com/gregjones/httpcache v0.0.0-20170920190843-316c5e0ff04e/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v0.0.0-20170914154624-68e816d1c783/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= +github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jpillora/backoff v0.0.0-20170918002102-8eab2debe79d/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.8.2 h1:Bx0qjetmNjdFXASH02NSAREKpiaDwkO1DRZ3dV2KCcs= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.7.4-0.20170902060319-8d7837e64d3c/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.0.10-0.20170816031813-ad5389df28cd/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.2/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/nwaples/rardecode v1.0.0 h1:r7vGuS5akxOnR4JQSkko62RJ1ReCMXxQRPtxsiFMBOs= -github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v1.0.1-0.20190307181833-2b18fe1d885e/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/panjf2000/ants v1.3.0 h1:8pQ+8leaLc9lys2viEEr8md0U4RN6uOSUCE9bOYjQ9M= github.com/panjf2000/ants v1.3.0/go.mod h1:AaACblRPzq35m1g3enqYcxspbbiOJJYaxU2wMpm1cXY= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pierrec/lz4 v2.3.0+incompatible h1:CZzRn4Ut9GbUkHlQ7jqBXeZQV41ZSKWFc302ZU6lUTk= -github.com/pierrec/lz4 v2.3.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4 v2.4.0+incompatible h1:06usnXXDNcPvCHDkmPpkidf4jTc52UKld7UPfqKatY4= -github.com/pierrec/lz4 v2.4.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= +github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/qiangxue/fasthttp-routing v0.0.0-20160225050629-6ccdc2a18d87 h1:u7uCM+HS2caoEKSPtSFQvvUDXQtqZdu3MYtF+QEw7vA= +github.com/qiangxue/fasthttp-routing v0.0.0-20160225050629-6ccdc2a18d87/go.mod h1:zwr0xP4ZJxwCS/g2d+AUOUwfq/j2NC7a1rK3F0ZbVYM= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/shirou/gopsutil v2.19.11+incompatible h1:lJHR0foqAjI4exXqWsU3DbH7bX1xvdhGdnXTIARA9W4= -github.com/shirou/gopsutil v2.19.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/spf13/afero v0.0.0-20170901052352-ee1bd8ee15a1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.1.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= +github.com/spf13/jwalterweatherman v0.0.0-20170901151539-12bd96e66386/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.1-0.20170901120850-7aff26db30c1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.0.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4= +github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.9.0 h1:hNpmUdy/+ZXYpGy0OBfm7K0UQTzb73W0T0U4iJIVrMw= +github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.3.0 h1:sFPn2GLc3poCkfrpIXGhBD2X0CMIo4Q/zSULXrj/+uc= @@ -173,8 +179,6 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEa go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -gocv.io/x/gocv v0.21.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -186,53 +190,64 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191119073136-fc4aabc6c914 h1:MlY3mEfbnWGmUi4rtHOtNnnnN4UJRGSyLPx+DXA5Sq4= -golang.org/x/net v0.0.0-20191119073136-fc4aabc6c914/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20170517211232-f52d1811a629/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191223224216-5a3cf8467b4e h1:z2Flw7sLy7DxaQi3zDOvI9X+Kb06+G9iZJlkEyHvujE= -golang.org/x/sys v0.0.0-20191223224216-5a3cf8467b4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191205225056-3393d29bb9fe h1:BEVcKURC7E0EF+vD1l52Jb3LOM5Iwu7OI5FpdPuU50o= golang.org/x/tools v0.0.0-20191205225056-3393d29bb9fe/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.0.0-20170921000349-586095a6e407/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20170918111702-1e559d0a00ee/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= @@ -241,20 +256,30 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 h1:yiW+nvdHb9LVqSHQBXfZCieqV4fzYhNBql77zY0ykqs= gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk= gopkg.in/validator.v2 v2.0.0-20191107172027-c3144fdedc21 h1:2QQcyaEBdpfjjYkF0MXc69jZbHb4IOYuXz2UwsmVM8k= gopkg.in/validator.v2 v2.0.0-20191107172027-c3144fdedc21/go.mod h1:o4V0GXN9/CAmCsvJ0oXYZvrZOe7syiDZSN1GWGZTGzc= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= -gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/api v0.0.0-20190819141258-3544db3b9e44/go.mod h1:AOxZTnaXR/xiarlQL0JUfwQPxjmKDvVYoRp58cA7lUo= +k8s.io/apimachinery v0.0.0-20190817020851-f2f3a405f61d/go.mod h1:3jediapYqJ2w1BFw7lAZPCx7scubsTfosqHkhXCWJKw= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=