Skip to content

Commit b27b65d

Browse files
hunjixinhunjixin
authored
feat: add arg to force update object , and a command to commit object (#151)
1 parent fac84ce commit b27b65d

16 files changed

+399
-141
lines changed

‎api/custom_response.go

+17
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package api
33
import (
44
"encoding/json"
55
"errors"
6+
"fmt"
67
"net/http"
78

89
"github.com/GitDataAI/jiaozifs/auth"
@@ -67,6 +68,13 @@ func (response *JiaozifsResponse) Error(err error) {
6768
return
6869
}
6970

71+
var codeErr ErrCode
72+
if errors.As(err, &codeErr) {
73+
response.WriteHeader(int(codeErr))
74+
_, _ = response.Write([]byte(err.Error()))
75+
return
76+
}
77+
7078
response.WriteHeader(http.StatusInternalServerError)
7179
_, _ = response.Write([]byte(err.Error()))
7280
}
@@ -89,3 +97,12 @@ func (response *JiaozifsResponse) String(msg string, code ...int) {
8997
func (response *JiaozifsResponse) Code(code int) {
9098
response.WriteHeader(code)
9199
}
100+
101+
type ErrCode int
102+
103+
func NewErrCode(code int) ErrCode {
104+
return ErrCode(code)
105+
}
106+
func (err ErrCode) Error() string {
107+
return fmt.Sprintf("code %d msg %s", err, http.StatusText(int(err)))
108+
}

‎api/custom_response_test.go

+11
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,17 @@ func TestJiaozifsResponse(t *testing.T) {
7777
jzResp.Error(fmt.Errorf("mock"))
7878
})
7979

80+
t.Run("err code", func(t *testing.T) {
81+
ctrl := gomock.NewController(t)
82+
resp := NewMockResponseWriter(ctrl)
83+
jzResp := JiaozifsResponse{resp}
84+
85+
resp.EXPECT().WriteHeader(http.StatusConflict)
86+
resp.EXPECT().Write([]byte("mock code 409 msg Conflict"))
87+
88+
jzResp.Error(fmt.Errorf("mock %w", ErrCode(http.StatusConflict)))
89+
})
90+
8091
t.Run("error not found", func(t *testing.T) {
8192
ctrl := gomock.NewController(t)
8293
resp := NewMockResponseWriter(ctrl)

‎api/jiaozifs.gen.go

+121-94
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/swagger.yml

+9
Original file line numberDiff line numberDiff line change
@@ -1239,6 +1239,13 @@ paths:
12391239
tags:
12401240
- objects
12411241
operationId: uploadObject
1242+
parameters:
1243+
- in: query
1244+
name: isReplace
1245+
description: indicate to replace existing object or not
1246+
allowEmptyValue: true
1247+
schema:
1248+
type: boolean
12421249
x-validation-exclude-body: true
12431250
requestBody:
12441251
content:
@@ -1265,6 +1272,8 @@ paths:
12651272
description: ValidationError
12661273
401:
12671274
description: Unauthorized ValidationError
1275+
409:
1276+
description: Resource Conflict
12681277
403:
12691278
description: Forbidden
12701279
404:

‎cmd/helper.go

+14-14
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,24 @@
11
package cmd
22

33
import (
4+
"io"
5+
"net/http"
6+
47
"github.com/GitDataAI/jiaozifs/api"
5-
"github.com/GitDataAI/jiaozifs/config"
8+
"github.com/spf13/cobra"
69
)
710

8-
func GetDefaultClient() (*api.Client, error) {
9-
swagger, err := api.GetSwagger()
10-
if err != nil {
11-
return nil, err
12-
}
11+
func GetClient(cmd *cobra.Command) (*api.Client, error) {
12+
url := cmd.Flags().Lookup("url").Value.String()
13+
ak := cmd.Flags().Lookup("ak").Value.String()
14+
sk := cmd.Flags().Lookup("sk").Value.String()
15+
return api.NewClient(url, api.AkSkOption(ak, sk))
16+
}
1317

14-
//get runtime version
15-
cfg, err := config.LoadConfig(cfgFile)
16-
if err != nil {
17-
return nil, err
18-
}
19-
basePath, err := swagger.Servers[0].BasePath()
18+
func tryLogError(resp *http.Response) string {
19+
bodyContent, err := io.ReadAll(resp.Body)
2020
if err != nil {
21-
return nil, err
21+
return ""
2222
}
23-
return api.NewClient(cfg.API.Listen + basePath)
23+
return string(bodyContent)
2424
}

‎cmd/root.go

+4
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,8 @@ func init() {
3535
_ = viper.BindPFlag("api.listen", rootCmd.PersistentFlags().Lookup("listen"))
3636
_ = viper.BindPFlag("config", rootCmd.PersistentFlags().Lookup("config"))
3737
_ = viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
38+
39+
uploadCmd.PersistentFlags().String("url", "https://127.0.0.1:34913", "url")
40+
uploadCmd.PersistentFlags().String("ak", "", "access key")
41+
uploadCmd.PersistentFlags().String("sk", "", "secret key")
3842
}

‎cmd/uploadfiles.go

+139
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
package cmd
2+
3+
import (
4+
"errors"
5+
"fmt"
6+
"io/fs"
7+
"net/http"
8+
"os"
9+
path2 "path"
10+
"path/filepath"
11+
"strings"
12+
13+
"github.com/GitDataAI/jiaozifs/utils"
14+
15+
"github.com/GitDataAI/jiaozifs/api"
16+
17+
"github.com/spf13/cobra"
18+
)
19+
20+
// versionCmd represents the version command
21+
var uploadCmd = &cobra.Command{
22+
Use: "upload",
23+
Short: "upload files to server",
24+
RunE: func(cmd *cobra.Command, _ []string) error {
25+
client, err := GetClient(cmd)
26+
if err != nil {
27+
return err
28+
}
29+
30+
path, err := cmd.Flags().GetString("path")
31+
if err != nil {
32+
return err
33+
}
34+
path = path2.Clean(path)
35+
if len(path) == 0 {
36+
return errors.New("path must be set")
37+
}
38+
39+
owner, err := cmd.Flags().GetString("owner")
40+
if err != nil {
41+
return err
42+
}
43+
if len(owner) == 0 {
44+
return errors.New("owner must be set")
45+
}
46+
47+
repo, err := cmd.Flags().GetString("repo")
48+
if err != nil {
49+
return err
50+
}
51+
if len(owner) == 0 || len(repo) == 0 {
52+
return errors.New("owner and repo must be set")
53+
}
54+
55+
refName, err := cmd.Flags().GetString("refName")
56+
if err != nil {
57+
return err
58+
}
59+
if len(refName) == 0 {
60+
return errors.New("refName must be set")
61+
}
62+
63+
uploadPath, err := cmd.Flags().GetString("uploadPath")
64+
if err != nil {
65+
return err
66+
}
67+
uploadPath = path2.Clean(uploadPath)
68+
if len(uploadPath) == 0 {
69+
uploadPath = "/"
70+
}
71+
72+
if len(path) == 0 {
73+
return errors.New("path not set")
74+
}
75+
76+
st, err := os.Stat(path)
77+
if err != nil {
78+
return err
79+
}
80+
81+
var files []string
82+
if st.IsDir() {
83+
err = filepath.Walk(path, func(path string, info fs.FileInfo, _ error) error {
84+
if info.IsDir() {
85+
return nil
86+
}
87+
files = append(files, path)
88+
return nil
89+
})
90+
if err != nil {
91+
return err
92+
}
93+
}
94+
95+
fmt.Printf("Files dected, %d files need to be uploaded\n", len(files))
96+
_, err = client.GetWip(cmd.Context(), owner, repo, &api.GetWipParams{RefName: refName})
97+
if err != nil {
98+
return err
99+
}
100+
101+
basename := filepath.Base(path)
102+
for _, file := range files {
103+
fs, err := os.Open(file)
104+
if err != nil {
105+
return err
106+
}
107+
relativePath := strings.Replace(file, path, "", 1)
108+
destPath := path2.Join(uploadPath, basename, relativePath)
109+
110+
resp, err := client.UploadObjectWithBody(cmd.Context(), owner, repo, &api.UploadObjectParams{
111+
RefName: refName,
112+
// Path relative to the ref
113+
Path: destPath,
114+
IsReplace: utils.Bool(true),
115+
}, "application/json", fs)
116+
if err != nil {
117+
return err
118+
}
119+
120+
if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusOK {
121+
fmt.Println("Upload file success ", file, " dest path", destPath)
122+
continue
123+
}
124+
return fmt.Errorf("upload file failed %d, %s", resp.StatusCode, tryLogError(resp))
125+
}
126+
return nil
127+
},
128+
}
129+
130+
func init() {
131+
rootCmd.AddCommand(uploadCmd)
132+
133+
uploadCmd.Flags().String("path", "", "path of files to upload")
134+
uploadCmd.Flags().String("owner", "", "owner")
135+
uploadCmd.Flags().String("repo", "", "repo")
136+
uploadCmd.Flags().String("refName", "main", "branch name")
137+
uploadCmd.Flags().String("uploadPath", "", "path to save in server")
138+
uploadCmd.Flags().Bool("replace", true, "path to save in server")
139+
}

‎cmd/version.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ var versionCmd = &cobra.Command{
2121
fmt.Println("Version ", version.UserVersion())
2222
fmt.Println("API Version ", swagger.Info.Version)
2323

24-
client, err := GetDefaultClient()
24+
client, err := GetClient(cmd)
2525
if err != nil {
2626
return err
2727
}

‎codecov.yml

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
ignore:
22
- "**/*.gen.go"
33
- "examples"
4+
- "cmd"

‎controller/object_ctl.go

+23-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package controller
22

33
import (
4+
"bytes"
45
"context"
56
"errors"
67
"fmt"
@@ -388,7 +389,28 @@ func (oct ObjectController) UploadObject(ctx context.Context, w *api.JiaozifsRes
388389

389390
path := versionmgr.CleanPath(params.Path)
390391
err = oct.Repo.Transaction(ctx, func(dRepo models.IRepo) error {
391-
err = workTree.AddLeaf(ctx, path, blob)
392+
oldData, _, err := workTree.FindBlob(ctx, path)
393+
if err != nil && !errors.Is(err, versionmgr.ErrPathNotFound) {
394+
return err
395+
}
396+
if oldData == nil {
397+
err = workTree.AddLeaf(ctx, path, blob)
398+
if err != nil {
399+
return err
400+
}
401+
return dRepo.WipRepo().UpdateByID(ctx, models.NewUpdateWipParams(workRepo.CurWip().ID).SetCurrentTree(workTree.Root().Hash()))
402+
}
403+
404+
if bytes.Equal(oldData.CheckSum, blob.CheckSum) {
405+
return nil
406+
}
407+
408+
if !utils.BoolValue(params.IsReplace) {
409+
return fmt.Errorf("object exit %w", api.ErrCode(http.StatusConflict))
410+
}
411+
412+
//allow to update
413+
err = workTree.ReplaceLeaf(ctx, path, blob)
392414
if err != nil {
393415
return err
394416
}

‎integrationtest/commit_test.go

+9-9
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ func GetEntriesInRefSpec(ctx context.Context, urlStr string) func(c convey.C) {
2626
_ = createRepo(ctx, client, repoName, false)
2727
_ = createBranch(ctx, client, userName, repoName, "main", branchName)
2828
_ = createWip(ctx, client, userName, repoName, branchName)
29-
_ = uploadObject(ctx, client, userName, repoName, branchName, "m.dat")
30-
_ = uploadObject(ctx, client, userName, repoName, branchName, "g/x.dat")
31-
_ = uploadObject(ctx, client, userName, repoName, branchName, "g/m.dat")
29+
_ = uploadObject(ctx, client, userName, repoName, branchName, "m.dat", true)
30+
_ = uploadObject(ctx, client, userName, repoName, branchName, "g/x.dat", true)
31+
_ = uploadObject(ctx, client, userName, repoName, branchName, "g/m.dat", true)
3232
})
3333

3434
c.Convey("get wip entries", func(c convey.C) {
@@ -230,8 +230,8 @@ func GetEntriesInRefSpec(ctx context.Context, urlStr string) func(c convey.C) {
230230

231231
c.Convey("prepare data for commit test", func(_ convey.C) {
232232
createWip(ctx, client, userName, repoName, "main")
233-
uploadObject(ctx, client, userName, repoName, "main", "a.dat") //delete\
234-
uploadObject(ctx, client, userName, repoName, "main", "g/m.dat") //modify
233+
uploadObject(ctx, client, userName, repoName, "main", "a.dat", true) //delete\
234+
uploadObject(ctx, client, userName, repoName, "main", "g/m.dat", true) //modify
235235
_ = commitWip(ctx, client, userName, repoName, "main", "test")
236236
})
237237

@@ -389,21 +389,21 @@ func GetCommitChangesSpec(ctx context.Context, urlStr string) func(c convey.C) {
389389
loginAndSwitch(ctx, client, userName, false)
390390
createRepo(ctx, client, repoName, false)
391391
createWip(ctx, client, userName, repoName, "main")
392-
uploadObject(ctx, client, userName, repoName, "main", "m.dat")
392+
uploadObject(ctx, client, userName, repoName, "main", "m.dat", true)
393393
_ = commitWip(ctx, client, userName, repoName, "main", "test")
394394

395-
uploadObject(ctx, client, userName, repoName, "main", "g/x.dat")
395+
uploadObject(ctx, client, userName, repoName, "main", "g/x.dat", true)
396396
_ = commitWip(ctx, client, userName, repoName, "main", "test")
397397

398398
//delete
399399
deleteObject(ctx, client, userName, repoName, "main", "g/x.dat")
400400

401401
//modify
402402
deleteObject(ctx, client, userName, repoName, "main", "m.dat")
403-
uploadObject(ctx, client, userName, repoName, "main", "m.dat")
403+
uploadObject(ctx, client, userName, repoName, "main", "m.dat", true)
404404

405405
//insert
406-
uploadObject(ctx, client, userName, repoName, "main", "g/m.dat")
406+
uploadObject(ctx, client, userName, repoName, "main", "g/m.dat", true)
407407
_ = commitWip(ctx, client, userName, repoName, "main", "test")
408408

409409
})

‎integrationtest/helper_test.go

+6-5
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ func InitCmd(ctx context.Context, jzHome string, listen string, db string) error
2929
buf := new(bytes.Buffer)
3030
cmd.RootCmd().SetOut(buf)
3131
cmd.RootCmd().SetErr(buf)
32-
cmd.RootCmd().SetArgs([]string{"init", "--listen", listen, "--db_debug", "true", "--db", db,
32+
cmd.RootCmd().SetArgs([]string{"init", "--listen", listen, "--db_debug", "false", "--db", db,
3333
"--config", fmt.Sprintf("%s/config.toml", jzHome), "--bs_path", fmt.Sprintf("%s/blockstore", jzHome)})
3434

3535
return cmd.RootCmd().ExecuteContext(ctx)
@@ -171,11 +171,12 @@ func createRepo(ctx context.Context, client *api.Client, repoName string, visibl
171171
return result.JSON201
172172
}
173173

174-
func uploadObject(ctx context.Context, client *api.Client, user string, repoName string, refName string, path string) *api.ObjectStats { //nolint
174+
func uploadObject(ctx context.Context, client *api.Client, user string, repoName string, refName string, path string, replace bool) *api.ObjectStats { //nolint
175175
resp, err := client.UploadObjectWithBody(ctx, user, repoName, &api.UploadObjectParams{
176-
RefName: refName,
177-
Path: path,
178-
}, "application/octet-stream", io.LimitReader(rand.Reader, 50))
176+
RefName: refName,
177+
Path: path,
178+
IsReplace: utils.Bool(replace),
179+
}, "application/octet-stream", io.LimitReader(rand.Reader, 100))
179180
convey.So(err, convey.ShouldBeNil)
180181
convey.So(resp.StatusCode, convey.ShouldEqual, http.StatusCreated)
181182

0 commit comments

Comments
 (0)