From 7161bcb4407b52e2de668836eac7c66dfc97567d Mon Sep 17 00:00:00 2001 From: hiifong Date: Sat, 14 Dec 2024 14:21:27 +0800 Subject: [PATCH 1/4] In some lfs server implementations, they require the ref attribute. --- modules/lfs/client.go | 2 +- modules/lfs/filesystem_client.go | 2 +- modules/lfs/http_client.go | 14 +++++++------- modules/repository/repo.go | 8 +++++++- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/modules/lfs/client.go b/modules/lfs/client.go index f810e5c7aae54..68074ff48b01f 100644 --- a/modules/lfs/client.go +++ b/modules/lfs/client.go @@ -19,7 +19,7 @@ type UploadCallback func(p Pointer, objectError error) (io.ReadCloser, error) // Client is used to communicate with a LFS source type Client interface { BatchSize() int - Download(ctx context.Context, objects []Pointer, callback DownloadCallback) error + Download(ctx context.Context, refName string, objects []Pointer, callback DownloadCallback) error Upload(ctx context.Context, objects []Pointer, callback UploadCallback) error } diff --git a/modules/lfs/filesystem_client.go b/modules/lfs/filesystem_client.go index 71bef5c899d6f..699ea8219e075 100644 --- a/modules/lfs/filesystem_client.go +++ b/modules/lfs/filesystem_client.go @@ -34,7 +34,7 @@ func (c *FilesystemClient) objectPath(oid string) string { } // Download reads the specific LFS object from the target path -func (c *FilesystemClient) Download(ctx context.Context, objects []Pointer, callback DownloadCallback) error { +func (c *FilesystemClient) Download(ctx context.Context, refName string, objects []Pointer, callback DownloadCallback) error { for _, object := range objects { p := Pointer{object.Oid, object.Size} diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index 50f0e7a8d88a6..156a04b5186ae 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -67,12 +67,12 @@ func (c *HTTPClient) transferNames() []string { return keys } -func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Pointer) (*BatchResponse, error) { +func (c *HTTPClient) batch(ctx context.Context, operation string, refName string, objects []Pointer) (*BatchResponse, error) { log.Trace("BATCH operation with objects: %v", objects) url := fmt.Sprintf("%s/objects/batch", c.endpoint) - request := &BatchRequest{operation, c.transferNames(), nil, objects} + request := &BatchRequest{operation, c.transferNames(), &Reference{Name: refName}, objects} payload := new(bytes.Buffer) err := json.NewEncoder(payload).Encode(request) if err != nil { @@ -106,17 +106,17 @@ func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Poin } // Download reads the specific LFS object from the LFS server -func (c *HTTPClient) Download(ctx context.Context, objects []Pointer, callback DownloadCallback) error { - return c.performOperation(ctx, objects, callback, nil) +func (c *HTTPClient) Download(ctx context.Context, refName string, objects []Pointer, callback DownloadCallback) error { + return c.performOperation(ctx, refName, objects, callback, nil) } // Upload sends the specific LFS object to the LFS server func (c *HTTPClient) Upload(ctx context.Context, objects []Pointer, callback UploadCallback) error { - return c.performOperation(ctx, objects, nil, callback) + return c.performOperation(ctx, "", objects, nil, callback) } // performOperation takes a slice of LFS object pointers, batches them, and performs the upload/download operations concurrently in each batch -func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc DownloadCallback, uc UploadCallback) error { +func (c *HTTPClient) performOperation(ctx context.Context, refName string, objects []Pointer, dc DownloadCallback, uc UploadCallback) error { if len(objects) == 0 { return nil } @@ -126,7 +126,7 @@ func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc operation = "upload" } - result, err := c.batch(ctx, operation, objects) + result, err := c.batch(ctx, operation, refName, objects) if err != nil { return err } diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 97b0343381327..fdcb4068d168d 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -179,8 +179,14 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Re errChan := make(chan error, 1) go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) + defaultBranch, err := gitrepo.GetDefaultBranch(ctx, repo) + if err != nil { + log.Error("Repo[%-v]: get default branch err: %v", repo, err) + return err + } + downloadObjects := func(pointers []lfs.Pointer) error { - err := lfsClient.Download(ctx, pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error { + err := lfsClient.Download(ctx, git.RefNameFromBranch(defaultBranch).String(), pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error { if errors.Is(objectError, lfs.ErrObjectNotExist) { log.Warn("Ignoring missing upstream LFS object %-v: %v", p, objectError) return nil From 7d40a362563349965808b1b6a92735b31f2615ab Mon Sep 17 00:00:00 2001 From: hiifong Date: Sat, 14 Dec 2024 14:36:45 +0800 Subject: [PATCH 2/4] Update --- modules/lfs/http_client.go | 2 +- modules/lfs/http_client_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index 156a04b5186ae..eb54d961ada24 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -67,7 +67,7 @@ func (c *HTTPClient) transferNames() []string { return keys } -func (c *HTTPClient) batch(ctx context.Context, operation string, refName string, objects []Pointer) (*BatchResponse, error) { +func (c *HTTPClient) batch(ctx context.Context, operation, refName string, objects []Pointer) (*BatchResponse, error) { log.Trace("BATCH operation with objects: %v", objects) url := fmt.Sprintf("%s/objects/batch", c.endpoint) diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go index aa7e3c45c4dcb..79de887c5d4ef 100644 --- a/modules/lfs/http_client_test.go +++ b/modules/lfs/http_client_test.go @@ -248,7 +248,7 @@ func TestHTTPClientDownload(t *testing.T) { }, } - err := client.Download(context.Background(), []Pointer{p}, func(p Pointer, content io.ReadCloser, objectError error) error { + err := client.Download(context.Background(), "", []Pointer{p}, func(p Pointer, content io.ReadCloser, objectError error) error { if objectError != nil { return objectError } From 7262a29a92f5c429dac4bdf0d072bb195d30b9d7 Mon Sep 17 00:00:00 2001 From: hiifong Date: Sat, 14 Dec 2024 14:46:57 +0800 Subject: [PATCH 3/4] Update --- modules/lfs/client.go | 2 +- modules/lfs/filesystem_client.go | 2 +- modules/lfs/http_client.go | 14 +++++++------- modules/lfs/http_client_test.go | 2 +- modules/repository/repo.go | 8 +------- 5 files changed, 11 insertions(+), 17 deletions(-) diff --git a/modules/lfs/client.go b/modules/lfs/client.go index 68074ff48b01f..f810e5c7aae54 100644 --- a/modules/lfs/client.go +++ b/modules/lfs/client.go @@ -19,7 +19,7 @@ type UploadCallback func(p Pointer, objectError error) (io.ReadCloser, error) // Client is used to communicate with a LFS source type Client interface { BatchSize() int - Download(ctx context.Context, refName string, objects []Pointer, callback DownloadCallback) error + Download(ctx context.Context, objects []Pointer, callback DownloadCallback) error Upload(ctx context.Context, objects []Pointer, callback UploadCallback) error } diff --git a/modules/lfs/filesystem_client.go b/modules/lfs/filesystem_client.go index 699ea8219e075..71bef5c899d6f 100644 --- a/modules/lfs/filesystem_client.go +++ b/modules/lfs/filesystem_client.go @@ -34,7 +34,7 @@ func (c *FilesystemClient) objectPath(oid string) string { } // Download reads the specific LFS object from the target path -func (c *FilesystemClient) Download(ctx context.Context, refName string, objects []Pointer, callback DownloadCallback) error { +func (c *FilesystemClient) Download(ctx context.Context, objects []Pointer, callback DownloadCallback) error { for _, object := range objects { p := Pointer{object.Oid, object.Size} diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index eb54d961ada24..534e2bfca62d6 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -67,12 +67,12 @@ func (c *HTTPClient) transferNames() []string { return keys } -func (c *HTTPClient) batch(ctx context.Context, operation, refName string, objects []Pointer) (*BatchResponse, error) { +func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Pointer) (*BatchResponse, error) { log.Trace("BATCH operation with objects: %v", objects) url := fmt.Sprintf("%s/objects/batch", c.endpoint) - request := &BatchRequest{operation, c.transferNames(), &Reference{Name: refName}, objects} + request := &BatchRequest{operation, c.transferNames(), &Reference{}, objects} payload := new(bytes.Buffer) err := json.NewEncoder(payload).Encode(request) if err != nil { @@ -106,17 +106,17 @@ func (c *HTTPClient) batch(ctx context.Context, operation, refName string, objec } // Download reads the specific LFS object from the LFS server -func (c *HTTPClient) Download(ctx context.Context, refName string, objects []Pointer, callback DownloadCallback) error { - return c.performOperation(ctx, refName, objects, callback, nil) +func (c *HTTPClient) Download(ctx context.Context, objects []Pointer, callback DownloadCallback) error { + return c.performOperation(ctx, objects, callback, nil) } // Upload sends the specific LFS object to the LFS server func (c *HTTPClient) Upload(ctx context.Context, objects []Pointer, callback UploadCallback) error { - return c.performOperation(ctx, "", objects, nil, callback) + return c.performOperation(ctx, objects, nil, callback) } // performOperation takes a slice of LFS object pointers, batches them, and performs the upload/download operations concurrently in each batch -func (c *HTTPClient) performOperation(ctx context.Context, refName string, objects []Pointer, dc DownloadCallback, uc UploadCallback) error { +func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc DownloadCallback, uc UploadCallback) error { if len(objects) == 0 { return nil } @@ -126,7 +126,7 @@ func (c *HTTPClient) performOperation(ctx context.Context, refName string, objec operation = "upload" } - result, err := c.batch(ctx, operation, refName, objects) + result, err := c.batch(ctx, operation, objects) if err != nil { return err } diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go index 79de887c5d4ef..aa7e3c45c4dcb 100644 --- a/modules/lfs/http_client_test.go +++ b/modules/lfs/http_client_test.go @@ -248,7 +248,7 @@ func TestHTTPClientDownload(t *testing.T) { }, } - err := client.Download(context.Background(), "", []Pointer{p}, func(p Pointer, content io.ReadCloser, objectError error) error { + err := client.Download(context.Background(), []Pointer{p}, func(p Pointer, content io.ReadCloser, objectError error) error { if objectError != nil { return objectError } diff --git a/modules/repository/repo.go b/modules/repository/repo.go index fdcb4068d168d..97b0343381327 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -179,14 +179,8 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Re errChan := make(chan error, 1) go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) - defaultBranch, err := gitrepo.GetDefaultBranch(ctx, repo) - if err != nil { - log.Error("Repo[%-v]: get default branch err: %v", repo, err) - return err - } - downloadObjects := func(pointers []lfs.Pointer) error { - err := lfsClient.Download(ctx, git.RefNameFromBranch(defaultBranch).String(), pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error { + err := lfsClient.Download(ctx, pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error { if errors.Is(objectError, lfs.ErrObjectNotExist) { log.Warn("Ignoring missing upstream LFS object %-v: %v", p, objectError) return nil From 9aae378e8d88b883ae01a8a12538b173aa4b75ec Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sun, 15 Dec 2024 09:41:18 +0800 Subject: [PATCH 4/4] Update modules/lfs/http_client.go --- modules/lfs/http_client.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index 534e2bfca62d6..3acd23b8f73cf 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -72,6 +72,9 @@ func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Poin url := fmt.Sprintf("%s/objects/batch", c.endpoint) + // `ref` is an "optional object describing the server ref that the objects belong to" + // but some (incorrect) lfs servers require it, so maybe adding an empty ref here doesn't break the correct ones. + // https://github.com/git-lfs/git-lfs/blob/a32a02b44bf8a511aa14f047627c49e1a7fd5021/docs/api/batch.md?plain=1#L37 request := &BatchRequest{operation, c.transferNames(), &Reference{}, objects} payload := new(bytes.Buffer) err := json.NewEncoder(payload).Encode(request)