From a179b31cd231a9eb7f590d8cd7fc78c430fc2931 Mon Sep 17 00:00:00 2001 From: ConcurrentCrab Date: Fri, 21 Jun 2024 17:38:44 +0000 Subject: [PATCH 1/8] routers/private: fix push-on-create being incorrectly triggered The code here checks if the repo being requested doesn't exist. If it doesn't, then a write operation might create it. But a read operation doesn't make any sense, and should error out. So simply check the access mode. I assume this was the intent here, but only checked for one "verb" instead, while there exist other read-only verbs as well. And ofc more can be introduced in the future ;) Possibly some write verbs don't make sense as well (presumably those that only add stuff incrementally to existing repos)? --- routers/private/serv.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/routers/private/serv.go b/routers/private/serv.go index dbb28cc2bb072..4dd7d06fb36e2 100644 --- a/routers/private/serv.go +++ b/routers/private/serv.go @@ -136,16 +136,15 @@ func ServCommand(ctx *context.PrivateContext) { if err != nil { if repo_model.IsErrRepoNotExist(err) { repoExist = false - for _, verb := range ctx.FormStrings("verb") { - if verb == "git-upload-pack" { - // User is fetching/cloning a non-existent repository - log.Warn("Failed authentication attempt (cannot find repository: %s/%s) from %s", results.OwnerName, results.RepoName, ctx.RemoteAddr()) - ctx.JSON(http.StatusNotFound, private.Response{ - UserMsg: fmt.Sprintf("Cannot find repository: %s/%s", results.OwnerName, results.RepoName), - }) - return - } + if mode == perm.AccessModeRead { + // User is fetching/cloning a non-existent repository + log.Warn("Failed authentication attempt (cannot find repository: %s/%s) from %s", results.OwnerName, results.RepoName, ctx.RemoteAddr()) + ctx.JSON(http.StatusNotFound, private.Response{ + UserMsg: fmt.Sprintf("Cannot find repository: %s/%s", results.OwnerName, results.RepoName), + }) + return } + // else fallthrough (push-to-create may kick in below) } else { log.Error("Unable to get repository: %s/%s Error: %v", results.OwnerName, results.RepoName, err) ctx.JSON(http.StatusInternalServerError, private.Response{ From f79825c19bc1961cdc8fa81dcc5cb42de1945da9 Mon Sep 17 00:00:00 2001 From: ConcurrentCrab Date: Sun, 30 Jun 2024 05:46:03 +0530 Subject: [PATCH 2/8] cmd: extract getLFSAuthToken() into function this makes no functional changes --- cmd/serv.go | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/cmd/serv.go b/cmd/serv.go index 2bfd1110617e5..33265119c09d3 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -124,6 +124,27 @@ func handleCliResponseExtra(extra private.ResponseExtra) error { return nil } +func getLFSAuthToken(ctx context.Context, lfsVerb string, results *private.ServCommandResults) (string, error) { + now := time.Now() + claims := lfs.Claims{ + RegisteredClaims: jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(now.Add(setting.LFS.HTTPAuthExpiry)), + NotBefore: jwt.NewNumericDate(now), + }, + RepoID: results.RepoID, + Op: lfsVerb, + UserID: results.UserID, + } + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + + // Sign and get the complete encoded token as a string using the secret + tokenString, err := token.SignedString(setting.LFS.JWTSecretBytes) + if err != nil { + return "", fail(ctx, "Failed to sign JWT Token", "Failed to sign JWT token: %v", err) + } + return fmt.Sprintf("Bearer %s", tokenString), nil +} + func runServ(c *cli.Context) error { ctx, cancel := installSignals() defer cancel() @@ -264,29 +285,16 @@ func runServ(c *cli.Context) error { if verb == lfsAuthenticateVerb { url := fmt.Sprintf("%s%s/%s.git/info/lfs", setting.AppURL, url.PathEscape(results.OwnerName), url.PathEscape(results.RepoName)) - now := time.Now() - claims := lfs.Claims{ - RegisteredClaims: jwt.RegisteredClaims{ - ExpiresAt: jwt.NewNumericDate(now.Add(setting.LFS.HTTPAuthExpiry)), - NotBefore: jwt.NewNumericDate(now), - }, - RepoID: results.RepoID, - Op: lfsVerb, - UserID: results.UserID, - } - token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) - - // Sign and get the complete encoded token as a string using the secret - tokenString, err := token.SignedString(setting.LFS.JWTSecretBytes) + token, err := getLFSAuthToken(ctx, lfsVerb, results) if err != nil { - return fail(ctx, "Failed to sign JWT Token", "Failed to sign JWT token: %v", err) + return err } tokenAuthentication := &git_model.LFSTokenResponse{ Header: make(map[string]string), Href: url, } - tokenAuthentication.Header["Authorization"] = fmt.Sprintf("Bearer %s", tokenString) + tokenAuthentication.Header["Authorization"] = token enc := json.NewEncoder(os.Stdout) err = enc.Encode(tokenAuthentication) From 6444a7cb4b8a7dca2b61a19f8e565702c3a04cd2 Mon Sep 17 00:00:00 2001 From: ConcurrentCrab Date: Fri, 21 Jun 2024 12:34:44 +0000 Subject: [PATCH 3/8] cmd: refactor runServ() Coalesce access mode detection into one place. Yes, "upload" really has opposite semantics for git commands vs. git-lfs commands. Wow. This commit makes no functional changes. --- cmd/serv.go | 73 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/cmd/serv.go b/cmd/serv.go index 33265119c09d3..41adf0f229a08 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -20,6 +20,7 @@ import ( asymkey_model "code.gitea.io/gitea/models/asymkey" git_model "code.gitea.io/gitea/models/git" "code.gitea.io/gitea/models/perm" + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" @@ -36,7 +37,10 @@ import ( ) const ( - lfsAuthenticateVerb = "git-lfs-authenticate" + verbUploadPack = "git-upload-pack" + verbUploadArchive = "git-upload-archive" + verbReceivePack = "git-receive-pack" + verbLfsAuthenticate = "git-lfs-authenticate" ) // CmdServ represents the available serv sub-command. @@ -73,12 +77,16 @@ func setup(ctx context.Context, debug bool) { } var ( - allowedCommands = map[string]perm.AccessMode{ - "git-upload-pack": perm.AccessModeRead, - "git-upload-archive": perm.AccessModeRead, - "git-receive-pack": perm.AccessModeWrite, - lfsAuthenticateVerb: perm.AccessModeNone, - } + // keep getAccessMode() in sync + allowedCommands = container.SetOf( + verbUploadPack, + verbUploadArchive, + verbReceivePack, + verbLfsAuthenticate, + ) + allowedCommandsLfs = container.SetOf( + verbLfsAuthenticate, + ) alphaDashDotPattern = regexp.MustCompile(`[^\w-\.]`) ) @@ -124,6 +132,24 @@ func handleCliResponseExtra(extra private.ResponseExtra) error { return nil } +func getAccessMode(verb, lfsVerb string) perm.AccessMode { + switch verb { + case verbUploadPack, verbUploadArchive: + return perm.AccessModeRead + case verbReceivePack: + return perm.AccessModeWrite + case verbLfsAuthenticate: + switch lfsVerb { + case "upload": + return perm.AccessModeWrite + case "download": + return perm.AccessModeRead + } + } + // should be unreachable + return perm.AccessModeNone +} + func getLFSAuthToken(ctx context.Context, lfsVerb string, results *private.ServCommandResults) (string, error) { now := time.Now() claims := lfs.Claims{ @@ -216,15 +242,6 @@ func runServ(c *cli.Context) error { } var lfsVerb string - if verb == lfsAuthenticateVerb { - if !setting.LFS.StartServer { - return fail(ctx, "Unknown git command", "LFS authentication request over SSH denied, LFS support is disabled") - } - - if len(words) > 2 { - lfsVerb = words[2] - } - } rr := strings.SplitN(repoPath, "/", 2) if len(rr) != 2 { @@ -261,20 +278,20 @@ func runServ(c *cli.Context) error { }() } - requestedMode, has := allowedCommands[verb] - if !has { + if allowedCommands.Contains(verb) { + if allowedCommandsLfs.Contains(verb) { + if !setting.LFS.StartServer { + return fail(ctx, "Unknown git command", "LFS authentication request over SSH denied, LFS support is disabled") + } + if len(words) > 2 { + lfsVerb = words[2] + } + } + } else { return fail(ctx, "Unknown git command", "Unknown git command %s", verb) } - if verb == lfsAuthenticateVerb { - if lfsVerb == "upload" { - requestedMode = perm.AccessModeWrite - } else if lfsVerb == "download" { - requestedMode = perm.AccessModeRead - } else { - return fail(ctx, "Unknown LFS verb", "Unknown lfs verb %s", lfsVerb) - } - } + requestedMode := getAccessMode(verb, lfsVerb) results, extra := private.ServCommand(ctx, keyID, username, reponame, requestedMode, verb, lfsVerb) if extra.HasError() { @@ -282,7 +299,7 @@ func runServ(c *cli.Context) error { } // LFS token authentication - if verb == lfsAuthenticateVerb { + if verb == verbLfsAuthenticate { url := fmt.Sprintf("%s%s/%s.git/info/lfs", setting.AppURL, url.PathEscape(results.OwnerName), url.PathEscape(results.RepoName)) token, err := getLFSAuthToken(ctx, lfsVerb, results) From bc7ae7633b8de87ce9b6b99984f8c212093d6ab3 Mon Sep 17 00:00:00 2001 From: ConcurrentCrab Date: Thu, 27 Jun 2024 08:45:03 +0530 Subject: [PATCH 4/8] modules/lfstransfer: add a backend and runner Also add handler in runServ() Doesn't add support for the locking API yet --- assets/go-licenses.json | 10 + cmd/serv.go | 15 +- go.mod | 4 + go.sum | 4 + modules/lfstransfer/backend/backend.go | 295 +++++++++++++++++++++++++ modules/lfstransfer/backend/util.go | 131 +++++++++++ modules/lfstransfer/logger.go | 21 ++ modules/lfstransfer/main.go | 42 ++++ 8 files changed, 521 insertions(+), 1 deletion(-) create mode 100644 modules/lfstransfer/backend/backend.go create mode 100644 modules/lfstransfer/backend/util.go create mode 100644 modules/lfstransfer/logger.go create mode 100644 modules/lfstransfer/main.go diff --git a/assets/go-licenses.json b/assets/go-licenses.json index 0181fd68ae703..62e63f271a838 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -344,6 +344,11 @@ "path": "github.com/cespare/xxhash/v2/LICENSE.txt", "licenseText": "Copyright (c) 2016 Caleb Spare\n\nMIT License\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n\"Software\"), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\nNONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE\nLIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\nWITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, + { + "name": "github.com/charmbracelet/git-lfs-transfer/transfer", + "path": "github.com/charmbracelet/git-lfs-transfer/transfer/LICENSE", + "licenseText": "MIT License\n\nCopyright (c) 2022-2023 Charmbracelet, Inc\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" + }, { "name": "github.com/chi-middleware/proxy", "path": "github.com/chi-middleware/proxy/LICENSE", @@ -464,6 +469,11 @@ "path": "github.com/fxamacker/cbor/v2/LICENSE", "licenseText": "MIT License\n\nCopyright (c) 2019-present Faye Amacker\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE." }, + { + "name": "github.com/git-lfs/pktline", + "path": "github.com/git-lfs/pktline/LICENSE.md", + "licenseText": "MIT License\n\nCopyright (c) 2014- GitHub, Inc. and Git LFS contributors\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n\nNote that Git LFS uses components from other Go modules (included in `vendor/`)\nwhich are under different licenses. See those LICENSE files for details.\n" + }, { "name": "github.com/gliderlabs/ssh", "path": "github.com/gliderlabs/ssh/LICENSE", diff --git a/cmd/serv.go b/cmd/serv.go index 41adf0f229a08..f7fcbc1967cad 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -23,6 +23,7 @@ import ( "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/lfstransfer" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/pprof" "code.gitea.io/gitea/modules/private" @@ -41,6 +42,7 @@ const ( verbUploadArchive = "git-upload-archive" verbReceivePack = "git-receive-pack" verbLfsAuthenticate = "git-lfs-authenticate" + verbLfsTransfer = "git-lfs-transfer" ) // CmdServ represents the available serv sub-command. @@ -83,9 +85,11 @@ var ( verbUploadArchive, verbReceivePack, verbLfsAuthenticate, + verbLfsTransfer, ) allowedCommandsLfs = container.SetOf( verbLfsAuthenticate, + verbLfsTransfer, ) alphaDashDotPattern = regexp.MustCompile(`[^\w-\.]`) ) @@ -138,7 +142,7 @@ func getAccessMode(verb, lfsVerb string) perm.AccessMode { return perm.AccessModeRead case verbReceivePack: return perm.AccessModeWrite - case verbLfsAuthenticate: + case verbLfsAuthenticate, verbLfsTransfer: switch lfsVerb { case "upload": return perm.AccessModeWrite @@ -298,6 +302,15 @@ func runServ(c *cli.Context) error { return fail(ctx, extra.UserMsg, "ServCommand failed: %s", extra.Error) } + // LFS SSH protocol + if verb == verbLfsTransfer { + token, err := getLFSAuthToken(ctx, lfsVerb, results) + if err != nil { + return err + } + return lfstransfer.Main(ctx, repoPath, lfsVerb, token) + } + // LFS token authentication if verb == verbLfsAuthenticate { url := fmt.Sprintf("%s%s/%s.git/info/lfs", setting.AppURL, url.PathEscape(results.OwnerName), url.PathEscape(results.RepoName)) diff --git a/go.mod b/go.mod index fae080f12f26d..fadbcc8b09b41 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/blevesearch/bleve/v2 v2.4.2 github.com/buildkite/terminal-to-html/v3 v3.12.1 github.com/caddyserver/certmagic v0.21.3 + github.com/charmbracelet/git-lfs-transfer v0.2.0 github.com/chi-middleware/proxy v1.1.1 github.com/dimiro1/reply v0.0.0-20200315094148-d0136a4c9e21 github.com/djherbis/buffer v1.2.0 @@ -192,6 +193,7 @@ require ( github.com/fatih/color v1.17.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.6.0 // indirect + github.com/git-lfs/pktline v0.0.0-20230103162542-ca444d533ef1 // indirect github.com/go-ap/errors v0.0.0-20240304112515-6077fa9c17b0 // indirect github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect github.com/go-enry/go-oniguruma v1.2.1 // indirect @@ -324,6 +326,8 @@ replace github.com/shurcooL/vfsgen => github.com/lunny/vfsgen v0.0.0-20220105142 replace github.com/nektos/act => gitea.com/gitea/act v0.259.1 +replace github.com/charmbracelet/git-lfs-transfer => gitea.com/gitea/git-lfs-transfer v0.2.0 + // TODO: This could be removed after https://github.com/mholt/archiver/pull/396 merged replace github.com/mholt/archiver/v3 => github.com/anchore/archiver/v3 v3.5.2 diff --git a/go.sum b/go.sum index 690e1301b761d..9a089e0f74e19 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078 h1:cliQ4H git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078/go.mod h1:g/V2Hjas6Z1UHUp4yIx6bATpNzJ7DYtD0FG3+xARWxs= gitea.com/gitea/act v0.259.1 h1:8GG1o/xtUHl3qjn5f0h/2FXrT5ubBn05TJOM5ry+FBw= gitea.com/gitea/act v0.259.1/go.mod h1:UxZWRYqQG2Yj4+4OqfGWW5a3HELwejyWFQyU7F1jUD8= +gitea.com/gitea/git-lfs-transfer v0.2.0 h1:baHaNoBSRaeq/xKayEXwiDQtlIjps4Ac/Ll4KqLMB40= +gitea.com/gitea/git-lfs-transfer v0.2.0/go.mod h1:UrXUCm3xLQkq15fu7qlXHUMlrhdlXHoi13KH2Dfiits= gitea.com/go-chi/binding v0.0.0-20240430071103-39a851e106ed h1:EZZBtilMLSZNWtHHcgq2mt6NSGhJSZBuduAlinMEmso= gitea.com/go-chi/binding v0.0.0-20240430071103-39a851e106ed/go.mod h1:E3i3cgB04dDx0v3CytCgRTTn9Z/9x891aet3r456RVw= gitea.com/go-chi/cache v0.2.1 h1:bfAPkvXlbcZxPCpcmDVCWoHgiBSBmZN/QosnZvEC0+g= @@ -291,6 +293,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.6.0 h1:sU6J2usfADwWlYDAFhZBQ6TnLFBHxgesMrQfQgk1tWA= github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/git-lfs/pktline v0.0.0-20230103162542-ca444d533ef1 h1:mtDjlmloH7ytdblogrMz1/8Hqua1y8B4ID+bh3rvod0= +github.com/git-lfs/pktline v0.0.0-20230103162542-ca444d533ef1/go.mod h1:fenKRzpXDjNpsIBhuhUzvjCKlDjKam0boRAenTE0Q6A= github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= diff --git a/modules/lfstransfer/backend/backend.go b/modules/lfstransfer/backend/backend.go new file mode 100644 index 0000000000000..dd1d642228bb3 --- /dev/null +++ b/modules/lfstransfer/backend/backend.go @@ -0,0 +1,295 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package backend + +import ( + "bytes" + "context" + "encoding/base64" + "io" + "net/http" + "net/url" + "strconv" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/setting" + + "github.com/charmbracelet/git-lfs-transfer/transfer" +) + +// Version is the git-lfs-transfer protocol version number. +const Version = "1" + +// Capabilities is a list of Git LFS capabilities supported by this package. +var Capabilities = []string{ + "version=" + Version, + // "locking", +} + +var _ transfer.Backend = &GiteaBackend{} + +// GiteaBackend is an adapter between git-lfs-transfer library and Gitea's internal LFS API +type GiteaBackend struct { + ctx context.Context + server *url.URL + op string + token string + logger transfer.Logger +} + +func New(ctx context.Context, repo, op, token string, logger transfer.Logger) (transfer.Backend, error) { + // runServ guarantees repo will be in form [owner]/[name].git + server, err := url.Parse(setting.LocalURL) + if err != nil { + return nil, err + } + server = server.JoinPath(repo, "info/lfs") + return &GiteaBackend{ctx: ctx, server: server, op: op, token: token, logger: logger}, nil +} + +// Batch implements transfer.Backend +func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args transfer.Args) ([]transfer.BatchItem, error) { + reqBody := lfs.BatchRequest{Operation: g.op} + if transfer, ok := args[argTransfer]; ok { + reqBody.Transfers = []string{transfer} + } + if ref, ok := args[argRefname]; ok { + reqBody.Ref = &lfs.Reference{Name: ref} + } + reqBody.Objects = make([]lfs.Pointer, len(pointers)) + for i := range pointers { + reqBody.Objects[i].Oid = pointers[i].Oid + reqBody.Objects[i].Size = pointers[i].Size + } + + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + g.logger.Log("json marshal error", err) + return nil, err + } + url := g.server.JoinPath("objects/batch").String() + headers := map[string]string{ + headerAuthorisation: g.token, + headerAccept: mimeGitLFS, + headerContentType: mimeGitLFS, + } + req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes) + resp, err := req.Response() + if err != nil { + g.logger.Log("http request error", err) + return nil, err + } + if resp.StatusCode != http.StatusOK { + g.logger.Log("http statuscode error", resp.StatusCode, statusCodeToErr(resp.StatusCode)) + return nil, statusCodeToErr(resp.StatusCode) + } + defer resp.Body.Close() + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + g.logger.Log("http read error", err) + return nil, err + } + var respBody lfs.BatchResponse + err = json.Unmarshal(respBytes, &respBody) + if err != nil { + g.logger.Log("json umarshal error", err) + return nil, err + } + + // rebuild slice, we can't rely on order in resp being the same as req + pointers = pointers[:0] + opNum := opMap[g.op] + for _, obj := range respBody.Objects { + pointer := transfer.Pointer{Oid: obj.Pointer.Oid, Size: obj.Pointer.Size} + item := transfer.BatchItem{Pointer: pointer, Args: map[string]string{}} + switch opNum { + case opDownload: + if action, ok := obj.Actions[actionDownload]; ok { + item.Present = true + idMap := obj.Actions + idMapBytes, err := json.Marshal(idMap) + if err != nil { + g.logger.Log("json marshal error", err) + return nil, err + } + idMapStr := base64.StdEncoding.EncodeToString(idMapBytes) + item.Args[argID] = idMapStr + if authHeader, ok := action.Header[headerAuthorisation]; ok { + authHeaderB64 := base64.StdEncoding.EncodeToString([]byte(authHeader)) + item.Args[argToken] = authHeaderB64 + } + if action.ExpiresAt != nil { + item.Args[argExpiresAt] = action.ExpiresAt.String() + } + } else { + // must be an error, but the SSH protocol can't propagate individual errors + g.logger.Log("object not found", obj.Pointer.Oid, obj.Pointer.Size) + item.Present = false + } + case opUpload: + if action, ok := obj.Actions[actionUpload]; ok { + item.Present = false + idMap := obj.Actions + idMapBytes, err := json.Marshal(idMap) + if err != nil { + g.logger.Log("json marshal error", err) + return nil, err + } + idMapStr := base64.StdEncoding.EncodeToString(idMapBytes) + item.Args[argID] = idMapStr + if authHeader, ok := action.Header[headerAuthorisation]; ok { + authHeaderB64 := base64.StdEncoding.EncodeToString([]byte(authHeader)) + item.Args[argToken] = authHeaderB64 + } + if action.ExpiresAt != nil { + item.Args[argExpiresAt] = action.ExpiresAt.String() + } + } else { + item.Present = true + } + } + pointers = append(pointers, item) + } + return pointers, nil +} + +// Download implements transfer.Backend. The returned reader must be closed by the +// caller. +func (g *GiteaBackend) Download(oid string, args transfer.Args) (io.ReadCloser, int64, error) { + idMapStr, exists := args[argID] + if !exists { + return nil, 0, ErrMissingID + } + idMapBytes, err := base64.StdEncoding.DecodeString(idMapStr) + if err != nil { + g.logger.Log("base64 decode error", err) + return nil, 0, transfer.ErrCorruptData + } + idMap := map[string]*lfs.Link{} + err = json.Unmarshal(idMapBytes, &idMap) + if err != nil { + g.logger.Log("json unmarshal error", err) + return nil, 0, transfer.ErrCorruptData + } + action, exists := idMap[actionDownload] + if !exists { + g.logger.Log("argument id incorrect") + return nil, 0, transfer.ErrCorruptData + } + url := action.Href + headers := map[string]string{ + headerAuthorisation: g.token, + headerAccept: mimeOctetStream, + } + req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil) + resp, err := req.Response() + if err != nil { + return nil, 0, err + } + if resp.StatusCode != http.StatusOK { + return nil, 0, statusCodeToErr(resp.StatusCode) + } + defer resp.Body.Close() + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, 0, err + } + respSize := int64(len(respBytes)) + respBuf := io.NopCloser(bytes.NewBuffer(respBytes)) + return respBuf, respSize, nil +} + +// StartUpload implements transfer.Backend. +func (g *GiteaBackend) Upload(oid string, size int64, r io.Reader, args transfer.Args) error { + idMapStr, exists := args[argID] + if !exists { + return ErrMissingID + } + idMapBytes, err := base64.StdEncoding.DecodeString(idMapStr) + if err != nil { + g.logger.Log("base64 decode error", err) + return transfer.ErrCorruptData + } + idMap := map[string]*lfs.Link{} + err = json.Unmarshal(idMapBytes, &idMap) + if err != nil { + g.logger.Log("json unmarshal error", err) + return transfer.ErrCorruptData + } + action, exists := idMap[actionUpload] + if !exists { + g.logger.Log("argument id incorrect") + return transfer.ErrCorruptData + } + url := action.Href + headers := map[string]string{ + headerAuthorisation: g.token, + headerContentType: mimeOctetStream, + headerContentLength: strconv.FormatInt(size, 10), + } + reqBytes, err := io.ReadAll(r) + if err != nil { + return err + } + req := newInternalRequest(g.ctx, url, http.MethodPut, headers, reqBytes) + resp, err := req.Response() + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return statusCodeToErr(resp.StatusCode) + } + return nil +} + +// Verify implements transfer.Backend. +func (g *GiteaBackend) Verify(oid string, size int64, args transfer.Args) (transfer.Status, error) { + reqBody := lfs.Pointer{Oid: oid, Size: size} + + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + return transfer.NewStatus(transfer.StatusInternalServerError), err + } + idMapStr, exists := args[argID] + if !exists { + return transfer.NewStatus(transfer.StatusBadRequest, "missing argument: id"), ErrMissingID + } + idMapBytes, err := base64.StdEncoding.DecodeString(idMapStr) + if err != nil { + g.logger.Log("base64 decode error", err) + return transfer.NewStatus(transfer.StatusBadRequest, "corrupt argument: id"), transfer.ErrCorruptData + } + idMap := map[string]*lfs.Link{} + err = json.Unmarshal(idMapBytes, &idMap) + if err != nil { + g.logger.Log("json unmarshal error", err) + return transfer.NewStatus(transfer.StatusBadRequest, "corrupt argument: id"), transfer.ErrCorruptData + } + action, exists := idMap[actionVerify] + if !exists { + // the server sent no verify action + return transfer.SuccessStatus(), nil + } + url := action.Href + headers := map[string]string{ + headerAuthorisation: g.token, + headerAccept: mimeGitLFS, + headerContentType: mimeGitLFS, + } + req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes) + resp, err := req.Response() + if err != nil { + return transfer.NewStatus(transfer.StatusInternalServerError), err + } + if resp.StatusCode != http.StatusOK { + return transfer.NewStatus(uint32(resp.StatusCode), http.StatusText(resp.StatusCode)), statusCodeToErr(resp.StatusCode) + } + return transfer.SuccessStatus(), nil +} + +// LockBackend implements transfer.Backend. +func (g *GiteaBackend) LockBackend(_ transfer.Args) transfer.LockBackend { + return (transfer.LockBackend)(nil) +} diff --git a/modules/lfstransfer/backend/util.go b/modules/lfstransfer/backend/util.go new file mode 100644 index 0000000000000..7329602cdab8a --- /dev/null +++ b/modules/lfstransfer/backend/util.go @@ -0,0 +1,131 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package backend + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "time" + + "code.gitea.io/gitea/modules/httplib" + "code.gitea.io/gitea/modules/proxyprotocol" + "code.gitea.io/gitea/modules/setting" + + "github.com/charmbracelet/git-lfs-transfer/transfer" +) + +// HTTP headers +const ( + headerAccept = "Accept" + headerAuthorisation = "Authorization" + headerContentType = "Content-Type" + headerContentLength = "Content-Length" +) + +// MIME types +const ( + mimeGitLFS = "application/vnd.git-lfs+json" + mimeOctetStream = "application/octet-stream" +) + +// SSH protocol action keys +const ( + actionDownload = "download" + actionUpload = "upload" + actionVerify = "verify" +) + +// SSH protocol argument keys +const ( + argExpiresAt = "expires-at" + argID = "id" + argRefname = "refname" + argToken = "token" + argTransfer = "transfer" +) + +// Operations enum +const ( + opNone = iota + opDownload + opUpload +) + +var opMap = map[string]int{ + "download": opDownload, + "upload": opUpload, +} + +var ErrMissingID = fmt.Errorf("%w: missing id arg", transfer.ErrMissingData) + +func statusCodeToErr(code int) error { + switch code { + case http.StatusBadRequest: + return transfer.ErrParseError + case http.StatusConflict: + return transfer.ErrConflict + case http.StatusForbidden: + return transfer.ErrForbidden + case http.StatusNotFound: + return transfer.ErrNotFound + case http.StatusUnauthorized: + return transfer.ErrUnauthorized + default: + return fmt.Errorf("server returned status %v: %v", code, http.StatusText(code)) + } +} + +func newInternalRequest(ctx context.Context, url, method string, headers map[string]string, body []byte) *httplib.Request { + req := httplib.NewRequest(url, method). + SetContext(ctx). + SetTimeout(10*time.Second, 60*time.Second). + SetTLSClientConfig(&tls.Config{ + InsecureSkipVerify: true, + }) + + if setting.Protocol == setting.HTTPUnix { + req.SetTransport(&http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + conn, err := d.DialContext(ctx, "unix", setting.HTTPAddr) + if err != nil { + return conn, err + } + if setting.LocalUseProxyProtocol { + if err = proxyprotocol.WriteLocalHeader(conn); err != nil { + _ = conn.Close() + return nil, err + } + } + return conn, err + }, + }) + } else if setting.LocalUseProxyProtocol { + req.SetTransport(&http.Transport{ + DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + var d net.Dialer + conn, err := d.DialContext(ctx, network, address) + if err != nil { + return conn, err + } + if err = proxyprotocol.WriteLocalHeader(conn); err != nil { + _ = conn.Close() + return nil, err + } + return conn, err + }, + }) + } + + for k, v := range headers { + req.Header(k, v) + } + + req.Body(body) + + return req +} diff --git a/modules/lfstransfer/logger.go b/modules/lfstransfer/logger.go new file mode 100644 index 0000000000000..517c2d9ba135e --- /dev/null +++ b/modules/lfstransfer/logger.go @@ -0,0 +1,21 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package lfstransfer + +import ( + "github.com/charmbracelet/git-lfs-transfer/transfer" +) + +var _ transfer.Logger = (*GiteaLogger)(nil) + +// noop logger for passing into transfer +type GiteaLogger struct{} + +func newLogger() transfer.Logger { + return &GiteaLogger{} +} + +// Log implements transfer.Logger +func (g *GiteaLogger) Log(msg string, itms ...any) { +} diff --git a/modules/lfstransfer/main.go b/modules/lfstransfer/main.go new file mode 100644 index 0000000000000..a134f50b86483 --- /dev/null +++ b/modules/lfstransfer/main.go @@ -0,0 +1,42 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package lfstransfer + +import ( + "context" + "fmt" + "os" + + "code.gitea.io/gitea/modules/lfstransfer/backend" + + "github.com/charmbracelet/git-lfs-transfer/transfer" +) + +func Main(ctx context.Context, repo, verb, token string) error { + logger := newLogger() + pktline := transfer.NewPktline(os.Stdin, os.Stdout, logger) + giteaBackend, err := backend.New(ctx, repo, verb, token, logger) + if err != nil { + return err + } + + for _, cap := range backend.Capabilities { + if err := pktline.WritePacketText(cap); err != nil { + logger.Log("error sending capability due to error:", err) + } + } + if err := pktline.WriteFlush(); err != nil { + logger.Log("error flushing capabilities:", err) + } + p := transfer.NewProcessor(pktline, giteaBackend, logger) + defer logger.Log("done processing commands") + switch verb { + case "upload": + return p.ProcessCommands(transfer.UploadOperation) + case "download": + return p.ProcessCommands(transfer.DownloadOperation) + default: + return fmt.Errorf("unknown operation %q", verb) + } +} From f819027108f752d588d1d3219dc830bd37f6a8ab Mon Sep 17 00:00:00 2001 From: ConcurrentCrab Date: Sun, 30 Jun 2024 22:53:43 +0530 Subject: [PATCH 5/8] modules/lfstransfer: add locking support --- modules/lfstransfer/backend/backend.go | 4 +- modules/lfstransfer/backend/lock.go | 292 +++++++++++++++++++++++++ modules/lfstransfer/backend/util.go | 9 + 3 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 modules/lfstransfer/backend/lock.go diff --git a/modules/lfstransfer/backend/backend.go b/modules/lfstransfer/backend/backend.go index dd1d642228bb3..e02b0a20271c7 100644 --- a/modules/lfstransfer/backend/backend.go +++ b/modules/lfstransfer/backend/backend.go @@ -25,7 +25,7 @@ const Version = "1" // Capabilities is a list of Git LFS capabilities supported by this package. var Capabilities = []string{ "version=" + Version, - // "locking", + "locking", } var _ transfer.Backend = &GiteaBackend{} @@ -291,5 +291,5 @@ func (g *GiteaBackend) Verify(oid string, size int64, args transfer.Args) (trans // LockBackend implements transfer.Backend. func (g *GiteaBackend) LockBackend(_ transfer.Args) transfer.LockBackend { - return (transfer.LockBackend)(nil) + return newGiteaLockBackend(g) } diff --git a/modules/lfstransfer/backend/lock.go b/modules/lfstransfer/backend/lock.go new file mode 100644 index 0000000000000..7c904617286f5 --- /dev/null +++ b/modules/lfstransfer/backend/lock.go @@ -0,0 +1,292 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package backend + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "time" + + "code.gitea.io/gitea/modules/json" + lfslock "code.gitea.io/gitea/modules/structs" + + "github.com/charmbracelet/git-lfs-transfer/transfer" +) + +var _ transfer.LockBackend = &giteaLockBackend{} + +type giteaLockBackend struct { + ctx context.Context + g *GiteaBackend + server *url.URL + token string + logger transfer.Logger +} + +func newGiteaLockBackend(g *GiteaBackend) transfer.LockBackend { + server := g.server.JoinPath("locks") + return &giteaLockBackend{ctx: g.ctx, g: g, server: server, token: g.token, logger: g.logger} +} + +// Create implements transfer.LockBackend +func (g *giteaLockBackend) Create(path, refname string) (transfer.Lock, error) { + reqBody := lfslock.LFSLockRequest{Path: path} + + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + g.logger.Log("json marshal error", err) + return nil, err + } + url := g.server.String() + headers := map[string]string{ + headerAuthorisation: g.token, + headerAccept: mimeGitLFS, + headerContentType: mimeGitLFS, + } + req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes) + resp, err := req.Response() + if err != nil { + g.logger.Log("http request error", err) + return nil, err + } + defer resp.Body.Close() + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + g.logger.Log("http read error", err) + return nil, err + } + if resp.StatusCode != http.StatusCreated { + g.logger.Log("http statuscode error", resp.StatusCode, statusCodeToErr(resp.StatusCode)) + return nil, statusCodeToErr(resp.StatusCode) + } + var respBody lfslock.LFSLockResponse + err = json.Unmarshal(respBytes, &respBody) + if err != nil { + g.logger.Log("json umarshal error", err) + return nil, err + } + + if respBody.Lock == nil { + g.logger.Log("api returned nil lock") + return nil, fmt.Errorf("api returned nil lock") + } + respLock := respBody.Lock + owner := userUnknown + if respLock.Owner != nil { + owner = respLock.Owner.Name + } + lock := newGiteaLock(g, respLock.ID, respLock.Path, respLock.LockedAt, owner) + return lock, nil +} + +// Unlock implements transfer.LockBackend +func (g *giteaLockBackend) Unlock(lock transfer.Lock) error { + reqBody := lfslock.LFSLockDeleteRequest{} + + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + g.logger.Log("json marshal error", err) + return err + } + url := g.server.JoinPath(lock.ID(), "unlock").String() + headers := map[string]string{ + headerAuthorisation: g.token, + headerAccept: mimeGitLFS, + headerContentType: mimeGitLFS, + } + req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes) + resp, err := req.Response() + if err != nil { + g.logger.Log("http request error", err) + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + g.logger.Log("http statuscode error", resp.StatusCode, statusCodeToErr(resp.StatusCode)) + return statusCodeToErr(resp.StatusCode) + } + // no need to read response + + return nil +} + +// FromPath implements transfer.LockBackend +func (g *giteaLockBackend) FromPath(path string) (transfer.Lock, error) { + v := url.Values{ + argPath: []string{path}, + } + + respLocks, _, err := g.queryLocks(v) + if err != nil { + return nil, err + } + + if len(respLocks) == 0 { + return nil, transfer.ErrNotFound + } + return respLocks[0], nil +} + +// FromID implements transfer.LockBackend +func (g *giteaLockBackend) FromID(id string) (transfer.Lock, error) { + v := url.Values{ + argID: []string{id}, + } + + respLocks, _, err := g.queryLocks(v) + if err != nil { + return nil, err + } + + if len(respLocks) == 0 { + return nil, transfer.ErrNotFound + } + return respLocks[0], nil +} + +// Range implements transfer.LockBackend +func (g *giteaLockBackend) Range(cursor string, limit int, iter func(transfer.Lock) error) (string, error) { + v := url.Values{ + argLimit: []string{strconv.FormatInt(int64(limit), 10)}, + } + if cursor != "" { + v[argCursor] = []string{cursor} + } + + respLocks, cursor, err := g.queryLocks(v) + if err != nil { + return "", err + } + + for _, lock := range respLocks { + err := iter(lock) + if err != nil { + return "", err + } + } + return cursor, nil +} + +func (g *giteaLockBackend) queryLocks(v url.Values) ([]transfer.Lock, string, error) { + urlq := g.server.JoinPath() // get a copy + urlq.RawQuery = v.Encode() + url := urlq.String() + headers := map[string]string{ + headerAuthorisation: g.token, + headerAccept: mimeGitLFS, + headerContentType: mimeGitLFS, + } + req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil) + resp, err := req.Response() + if err != nil { + g.logger.Log("http request error", err) + return nil, "", err + } + defer resp.Body.Close() + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + g.logger.Log("http read error", err) + return nil, "", err + } + if resp.StatusCode != http.StatusOK { + g.logger.Log("http statuscode error", resp.StatusCode, statusCodeToErr(resp.StatusCode)) + return nil, "", statusCodeToErr(resp.StatusCode) + } + var respBody lfslock.LFSLockList + err = json.Unmarshal(respBytes, &respBody) + if err != nil { + g.logger.Log("json umarshal error", err) + return nil, "", err + } + + respLocks := make([]transfer.Lock, 0, len(respBody.Locks)) + for _, respLock := range respBody.Locks { + owner := userUnknown + if respLock.Owner != nil { + owner = respLock.Owner.Name + } + lock := newGiteaLock(g, respLock.ID, respLock.Path, respLock.LockedAt, owner) + respLocks = append(respLocks, lock) + } + return respLocks, respBody.Next, nil +} + +var _ transfer.Lock = &giteaLock{} + +type giteaLock struct { + g *giteaLockBackend + id string + path string + lockedAt time.Time + owner string +} + +func newGiteaLock(g *giteaLockBackend, id, path string, lockedAt time.Time, owner string) transfer.Lock { + return &giteaLock{g: g, id: id, path: path, lockedAt: lockedAt, owner: owner} +} + +// Unlock implements transfer.Lock +func (g *giteaLock) Unlock() error { + return g.g.Unlock(g) +} + +// ID implements transfer.Lock +func (g *giteaLock) ID() string { + return g.id +} + +// Path implements transfer.Lock +func (g *giteaLock) Path() string { + return g.path +} + +// FormattedTimestamp implements transfer.Lock +func (g *giteaLock) FormattedTimestamp() string { + return g.lockedAt.UTC().Format(time.RFC3339) +} + +// OwnerName implements transfer.Lock +func (g *giteaLock) OwnerName() string { + return g.owner +} + +func (g *giteaLock) CurrentUser() (string, error) { + return userSelf, nil +} + +// AsLockSpec implements transfer.Lock +func (g *giteaLock) AsLockSpec(ownerID bool) ([]string, error) { + msgs := []string{ + fmt.Sprintf("lock %s", g.ID()), + fmt.Sprintf("path %s %s", g.ID(), g.Path()), + fmt.Sprintf("locked-at %s %s", g.ID(), g.FormattedTimestamp()), + fmt.Sprintf("ownername %s %s", g.ID(), g.OwnerName()), + } + if ownerID { + user, err := g.CurrentUser() + if err != nil { + return nil, fmt.Errorf("error getting current user: %w", err) + } + who := "theirs" + if user == g.OwnerName() { + who = "ours" + } + msgs = append(msgs, fmt.Sprintf("owner %s %s", g.ID(), who)) + } + return msgs, nil +} + +// AsArguments implements transfer.Lock +func (g *giteaLock) AsArguments() []string { + return []string{ + fmt.Sprintf("id=%s", g.ID()), + fmt.Sprintf("path=%s", g.Path()), + fmt.Sprintf("locked-at=%s", g.FormattedTimestamp()), + fmt.Sprintf("ownername=%s", g.OwnerName()), + } +} diff --git a/modules/lfstransfer/backend/util.go b/modules/lfstransfer/backend/util.go index 7329602cdab8a..2f20b362cca07 100644 --- a/modules/lfstransfer/backend/util.go +++ b/modules/lfstransfer/backend/util.go @@ -41,13 +41,22 @@ const ( // SSH protocol argument keys const ( + argCursor = "cursor" argExpiresAt = "expires-at" argID = "id" + argLimit = "limit" + argPath = "path" argRefname = "refname" argToken = "token" argTransfer = "transfer" ) +// Default username constants +const ( + userSelf = "(self)" + userUnknown = "(unknown)" +) + // Operations enum const ( opNone = iota From 5009913b9bd8b46936c6bf13fc58f5487cc7d87b Mon Sep 17 00:00:00 2001 From: ConcurrentCrab Date: Thu, 25 Jul 2024 21:43:08 +0530 Subject: [PATCH 6/8] config: add ALLOW_PURE_SSH option --- cmd/serv.go | 3 +++ custom/conf/app.example.ini | 2 ++ modules/setting/lfs.go | 1 + 3 files changed, 6 insertions(+) diff --git a/cmd/serv.go b/cmd/serv.go index f7fcbc1967cad..8bd2f6c65c104 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -287,6 +287,9 @@ func runServ(c *cli.Context) error { if !setting.LFS.StartServer { return fail(ctx, "Unknown git command", "LFS authentication request over SSH denied, LFS support is disabled") } + if verb == verbLfsTransfer && !setting.LFS.AllowPureSSH { + return fail(ctx, "Unknown git command", "LFS SSH transfer connection denied, pure SSH protocol is disabled") + } if len(words) > 2 { lfsVerb = words[2] } diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index a2dd92b1051ab..4304122ed33cb 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -306,6 +306,8 @@ RUN_USER = ; git ;; Enables git-lfs support. true or false, default is false. ;LFS_START_SERVER = false ;; +;; Enables git-lfs SSH protocol support. true or false, default is false. +;LFS_ALLOW_PURE_SSH = false ;; ;; LFS authentication secret, change this yourself ;LFS_JWT_SECRET = diff --git a/modules/setting/lfs.go b/modules/setting/lfs.go index 2034ef782c22c..6bdcbed91d90f 100644 --- a/modules/setting/lfs.go +++ b/modules/setting/lfs.go @@ -13,6 +13,7 @@ import ( // LFS represents the configuration for Git LFS var LFS = struct { StartServer bool `ini:"LFS_START_SERVER"` + AllowPureSSH bool `ini:"LFS_ALLOW_PURE_SSH"` JWTSecretBytes []byte `ini:"-"` HTTPAuthExpiry time.Duration `ini:"LFS_HTTP_AUTH_EXPIRY"` MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"` From 165bf9a658f75962f0ef95ac506735b4bdc2e12c Mon Sep 17 00:00:00 2001 From: ConcurrentCrab Date: Thu, 19 Sep 2024 15:52:37 +0530 Subject: [PATCH 7/8] routers/private: add private equivalents of public lfs API --- routers/private/internal.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/routers/private/internal.go b/routers/private/internal.go index 61e604b7a932e..28e5583393f98 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -12,7 +12,9 @@ import ( "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/routers/common" "code.gitea.io/gitea/services/context" + "code.gitea.io/gitea/services/lfs" "gitea.com/go-chi/binding" chi_middleware "github.com/go-chi/chi/v5/middleware" @@ -80,5 +82,25 @@ func Routes() *web.Router { r.Post("/restore_repo", RestoreRepo) r.Post("/actions/generate_actions_runner_token", GenerateActionsRunnerToken) + r.Group("/repo/{username}/{reponame}", func() { + r.Group("/info/lfs", func() { + r.Post("/objects/batch", lfs.CheckAcceptMediaType, lfs.BatchHandler) + r.Put("/objects/{oid}/{size}", lfs.UploadHandler) + r.Get("/objects/{oid}/{filename}", lfs.DownloadHandler) + r.Get("/objects/{oid}", lfs.DownloadHandler) + r.Post("/verify", lfs.CheckAcceptMediaType, lfs.VerifyHandler) + r.Group("/locks", func() { + r.Get("/", lfs.GetListLockHandler) + r.Post("/", lfs.PostLockHandler) + r.Post("/verify", lfs.VerifyLockHandler) + r.Post("/{lid}/unlock", lfs.UnLockHandler) + }, lfs.CheckAcceptMediaType) + r.Any("/*", func(ctx *context.Context) { + ctx.NotFound("", nil) + }) + }) + }, common.Sessioner(), context.Contexter()) + // end "/repo/{username}/{reponame}": git (LFS) API mirror + return r } From 1e8502fb99e1450b46885803a3459a85883762d9 Mon Sep 17 00:00:00 2001 From: ConcurrentCrab Date: Thu, 19 Sep 2024 07:49:19 +0000 Subject: [PATCH 8/8] modules/lfstransfer: call private LFS API --- modules/lfstransfer/backend/backend.go | 18 ++++++++++++------ modules/lfstransfer/backend/lock.go | 12 ++++++++---- modules/lfstransfer/backend/util.go | 1 + routers/private/internal.go | 10 +++++++++- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/modules/lfstransfer/backend/backend.go b/modules/lfstransfer/backend/backend.go index e02b0a20271c7..d4523e1abfab5 100644 --- a/modules/lfstransfer/backend/backend.go +++ b/modules/lfstransfer/backend/backend.go @@ -7,6 +7,7 @@ import ( "bytes" "context" "encoding/base64" + "fmt" "io" "net/http" "net/url" @@ -36,6 +37,7 @@ type GiteaBackend struct { server *url.URL op string token string + itoken string logger transfer.Logger } @@ -45,8 +47,8 @@ func New(ctx context.Context, repo, op, token string, logger transfer.Logger) (t if err != nil { return nil, err } - server = server.JoinPath(repo, "info/lfs") - return &GiteaBackend{ctx: ctx, server: server, op: op, token: token, logger: logger}, nil + server = server.JoinPath("api/internal/repo", repo, "info/lfs") + return &GiteaBackend{ctx: ctx, server: server, op: op, token: token, itoken: fmt.Sprintf("Bearer %s", setting.InternalToken), logger: logger}, nil } // Batch implements transfer.Backend @@ -71,7 +73,8 @@ func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args trans } url := g.server.JoinPath("objects/batch").String() headers := map[string]string{ - headerAuthorisation: g.token, + headerAuthorisation: g.itoken, + headerAuthX: g.token, headerAccept: mimeGitLFS, headerContentType: mimeGitLFS, } @@ -180,7 +183,8 @@ func (g *GiteaBackend) Download(oid string, args transfer.Args) (io.ReadCloser, } url := action.Href headers := map[string]string{ - headerAuthorisation: g.token, + headerAuthorisation: g.itoken, + headerAuthX: g.token, headerAccept: mimeOctetStream, } req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil) @@ -225,7 +229,8 @@ func (g *GiteaBackend) Upload(oid string, size int64, r io.Reader, args transfer } url := action.Href headers := map[string]string{ - headerAuthorisation: g.token, + headerAuthorisation: g.itoken, + headerAuthX: g.token, headerContentType: mimeOctetStream, headerContentLength: strconv.FormatInt(size, 10), } @@ -274,7 +279,8 @@ func (g *GiteaBackend) Verify(oid string, size int64, args transfer.Args) (trans } url := action.Href headers := map[string]string{ - headerAuthorisation: g.token, + headerAuthorisation: g.itoken, + headerAuthX: g.token, headerAccept: mimeGitLFS, headerContentType: mimeGitLFS, } diff --git a/modules/lfstransfer/backend/lock.go b/modules/lfstransfer/backend/lock.go index 7c904617286f5..f72ffd5b6f96d 100644 --- a/modules/lfstransfer/backend/lock.go +++ b/modules/lfstransfer/backend/lock.go @@ -25,12 +25,13 @@ type giteaLockBackend struct { g *GiteaBackend server *url.URL token string + itoken string logger transfer.Logger } func newGiteaLockBackend(g *GiteaBackend) transfer.LockBackend { server := g.server.JoinPath("locks") - return &giteaLockBackend{ctx: g.ctx, g: g, server: server, token: g.token, logger: g.logger} + return &giteaLockBackend{ctx: g.ctx, g: g, server: server, token: g.token, itoken: g.itoken, logger: g.logger} } // Create implements transfer.LockBackend @@ -44,7 +45,8 @@ func (g *giteaLockBackend) Create(path, refname string) (transfer.Lock, error) { } url := g.server.String() headers := map[string]string{ - headerAuthorisation: g.token, + headerAuthorisation: g.itoken, + headerAuthX: g.token, headerAccept: mimeGitLFS, headerContentType: mimeGitLFS, } @@ -95,7 +97,8 @@ func (g *giteaLockBackend) Unlock(lock transfer.Lock) error { } url := g.server.JoinPath(lock.ID(), "unlock").String() headers := map[string]string{ - headerAuthorisation: g.token, + headerAuthorisation: g.itoken, + headerAuthX: g.token, headerAccept: mimeGitLFS, headerContentType: mimeGitLFS, } @@ -177,7 +180,8 @@ func (g *giteaLockBackend) queryLocks(v url.Values) ([]transfer.Lock, string, er urlq.RawQuery = v.Encode() url := urlq.String() headers := map[string]string{ - headerAuthorisation: g.token, + headerAuthorisation: g.itoken, + headerAuthX: g.token, headerAccept: mimeGitLFS, headerContentType: mimeGitLFS, } diff --git a/modules/lfstransfer/backend/util.go b/modules/lfstransfer/backend/util.go index 2f20b362cca07..126ac001753db 100644 --- a/modules/lfstransfer/backend/util.go +++ b/modules/lfstransfer/backend/util.go @@ -22,6 +22,7 @@ import ( const ( headerAccept = "Accept" headerAuthorisation = "Authorization" + headerAuthX = "X-Auth" headerContentType = "Content-Type" headerContentLength = "Content-Length" ) diff --git a/routers/private/internal.go b/routers/private/internal.go index 28e5583393f98..f9adff388cfd0 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -48,6 +48,14 @@ func bind[T any](_ T) any { } } +// SwapAuthToken swaps Authorization header with X-Auth header +func swapAuthToken(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + req.Header.Set("Authorization", req.Header.Get("X-Auth")) + next.ServeHTTP(w, req) + }) +} + // Routes registers all internal APIs routes to web application. // These APIs will be invoked by internal commands for example `gitea serv` and etc. func Routes() *web.Router { @@ -98,7 +106,7 @@ func Routes() *web.Router { r.Any("/*", func(ctx *context.Context) { ctx.NotFound("", nil) }) - }) + }, swapAuthToken) }, common.Sessioner(), context.Contexter()) // end "/repo/{username}/{reponame}": git (LFS) API mirror