Skip to content

Commit 8176867

Browse files
wxiaoguangsschroe
wxiaoguang
and
sschroe
authored
Inherit submodules from template repository content (go-gitea#16237) (go-gitea#33068)
Backport go-gitea#16237 (it more likely a bug fix) Co-authored-by: Steffen Schröter <[email protected]>
1 parent 39cc725 commit 8176867

17 files changed

+289
-134
lines changed

Diff for: ‎modules/git/batch_reader.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -253,15 +253,15 @@ func BinToHex(objectFormat ObjectFormat, sha, out []byte) []byte {
253253
return out
254254
}
255255

256-
// ParseTreeLine reads an entry from a tree in a cat-file --batch stream
256+
// ParseCatFileTreeLine reads an entry from a tree in a cat-file --batch stream
257257
// This carefully avoids allocations - except where fnameBuf is too small.
258258
// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations
259259
//
260260
// Each line is composed of:
261261
// <mode-in-ascii-dropping-initial-zeros> SP <fname> NUL <binary HASH>
262262
//
263263
// We don't attempt to convert the raw HASH to save a lot of time
264-
func ParseTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) {
264+
func ParseCatFileTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) {
265265
var readBytes []byte
266266

267267
// Read the Mode & fname
@@ -271,7 +271,7 @@ func ParseTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBu
271271
}
272272
idx := bytes.IndexByte(readBytes, ' ')
273273
if idx < 0 {
274-
log.Debug("missing space in readBytes ParseTreeLine: %s", readBytes)
274+
log.Debug("missing space in readBytes ParseCatFileTreeLine: %s", readBytes)
275275
return mode, fname, sha, n, &ErrNotExist{}
276276
}
277277

Diff for: ‎modules/git/parse.go

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// Copyright 2024 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package git
5+
6+
import (
7+
"bytes"
8+
"fmt"
9+
"strconv"
10+
"strings"
11+
12+
"code.gitea.io/gitea/modules/optional"
13+
)
14+
15+
var sepSpace = []byte{' '}
16+
17+
type LsTreeEntry struct {
18+
ID ObjectID
19+
EntryMode EntryMode
20+
Name string
21+
Size optional.Option[int64]
22+
}
23+
24+
func parseLsTreeLine(line []byte) (*LsTreeEntry, error) {
25+
// expect line to be of the form:
26+
// <mode> <type> <sha> <space-padded-size>\t<filename>
27+
// <mode> <type> <sha>\t<filename>
28+
29+
var err error
30+
posTab := bytes.IndexByte(line, '\t')
31+
if posTab == -1 {
32+
return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line)
33+
}
34+
35+
entry := new(LsTreeEntry)
36+
37+
entryAttrs := line[:posTab]
38+
entryName := line[posTab+1:]
39+
40+
entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
41+
_ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type
42+
entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
43+
if len(entryAttrs) > 0 {
44+
entrySize := entryAttrs // the last field is the space-padded-size
45+
size, _ := strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64)
46+
entry.Size = optional.Some(size)
47+
}
48+
49+
switch string(entryMode) {
50+
case "100644":
51+
entry.EntryMode = EntryModeBlob
52+
case "100755":
53+
entry.EntryMode = EntryModeExec
54+
case "120000":
55+
entry.EntryMode = EntryModeSymlink
56+
case "160000":
57+
entry.EntryMode = EntryModeCommit
58+
case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons
59+
entry.EntryMode = EntryModeTree
60+
default:
61+
return nil, fmt.Errorf("unknown type: %v", string(entryMode))
62+
}
63+
64+
entry.ID, err = NewIDFromString(string(entryObjectID))
65+
if err != nil {
66+
return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err)
67+
}
68+
69+
if len(entryName) > 0 && entryName[0] == '"' {
70+
entry.Name, err = strconv.Unquote(string(entryName))
71+
if err != nil {
72+
return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err)
73+
}
74+
} else {
75+
entry.Name = string(entryName)
76+
}
77+
return entry, nil
78+
}

Diff for: ‎modules/git/parse_nogogit.go

+12-55
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,6 @@ import (
1010
"bytes"
1111
"fmt"
1212
"io"
13-
"strconv"
14-
"strings"
1513

1614
"code.gitea.io/gitea/modules/log"
1715
)
@@ -21,71 +19,30 @@ func ParseTreeEntries(data []byte) ([]*TreeEntry, error) {
2119
return parseTreeEntries(data, nil)
2220
}
2321

24-
var sepSpace = []byte{' '}
25-
22+
// parseTreeEntries FIXME this function's design is not right, it should make the caller read all data into memory
2623
func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
27-
var err error
2824
entries := make([]*TreeEntry, 0, bytes.Count(data, []byte{'\n'})+1)
2925
for pos := 0; pos < len(data); {
30-
// expect line to be of the form:
31-
// <mode> <type> <sha> <space-padded-size>\t<filename>
32-
// <mode> <type> <sha>\t<filename>
3326
posEnd := bytes.IndexByte(data[pos:], '\n')
3427
if posEnd == -1 {
3528
posEnd = len(data)
3629
} else {
3730
posEnd += pos
3831
}
39-
line := data[pos:posEnd]
40-
posTab := bytes.IndexByte(line, '\t')
41-
if posTab == -1 {
42-
return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line)
43-
}
44-
45-
entry := new(TreeEntry)
46-
entry.ptree = ptree
47-
48-
entryAttrs := line[:posTab]
49-
entryName := line[posTab+1:]
50-
51-
entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
52-
_ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type
53-
entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
54-
if len(entryAttrs) > 0 {
55-
entrySize := entryAttrs // the last field is the space-padded-size
56-
entry.size, _ = strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64)
57-
entry.sized = true
58-
}
5932

60-
switch string(entryMode) {
61-
case "100644":
62-
entry.entryMode = EntryModeBlob
63-
case "100755":
64-
entry.entryMode = EntryModeExec
65-
case "120000":
66-
entry.entryMode = EntryModeSymlink
67-
case "160000":
68-
entry.entryMode = EntryModeCommit
69-
case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons
70-
entry.entryMode = EntryModeTree
71-
default:
72-
return nil, fmt.Errorf("unknown type: %v", string(entryMode))
73-
}
74-
75-
entry.ID, err = NewIDFromString(string(entryObjectID))
33+
line := data[pos:posEnd]
34+
lsTreeLine, err := parseLsTreeLine(line)
7635
if err != nil {
77-
return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err)
36+
return nil, err
7837
}
79-
80-
if len(entryName) > 0 && entryName[0] == '"' {
81-
entry.name, err = strconv.Unquote(string(entryName))
82-
if err != nil {
83-
return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err)
84-
}
85-
} else {
86-
entry.name = string(entryName)
38+
entry := &TreeEntry{
39+
ptree: ptree,
40+
ID: lsTreeLine.ID,
41+
entryMode: lsTreeLine.EntryMode,
42+
name: lsTreeLine.Name,
43+
size: lsTreeLine.Size.Value(),
44+
sized: lsTreeLine.Size.Has(),
8745
}
88-
8946
pos = posEnd + 1
9047
entries = append(entries, entry)
9148
}
@@ -100,7 +57,7 @@ func catBatchParseTreeEntries(objectFormat ObjectFormat, ptree *Tree, rd *bufio.
10057

10158
loop:
10259
for sz > 0 {
103-
mode, fname, sha, count, err := ParseTreeLine(objectFormat, rd, modeBuf, fnameBuf, shaBuf)
60+
mode, fname, sha, count, err := ParseCatFileTreeLine(objectFormat, rd, modeBuf, fnameBuf, shaBuf)
10461
if err != nil {
10562
if err == io.EOF {
10663
break loop

Diff for: ‎modules/git/pipeline/lfs_nogogit.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
114114
case "tree":
115115
var n int64
116116
for n < size {
117-
mode, fname, binObjectID, count, err := git.ParseTreeLine(objectID.Type(), batchReader, modeBuf, fnameBuf, workingShaBuf)
117+
mode, fname, binObjectID, count, err := git.ParseCatFileTreeLine(objectID.Type(), batchReader, modeBuf, fnameBuf, workingShaBuf)
118118
if err != nil {
119119
return nil, err
120120
}

Diff for: ‎modules/git/submodule.go

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
// Copyright 2024 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package git
5+
6+
import (
7+
"bufio"
8+
"context"
9+
"fmt"
10+
"os"
11+
12+
"code.gitea.io/gitea/modules/log"
13+
)
14+
15+
type TemplateSubmoduleCommit struct {
16+
Path string
17+
Commit string
18+
}
19+
20+
// GetTemplateSubmoduleCommits returns a list of submodules paths and their commits from a repository
21+
// This function is only for generating new repos based on existing template, the template couldn't be too large.
22+
func GetTemplateSubmoduleCommits(ctx context.Context, repoPath string) (submoduleCommits []TemplateSubmoduleCommit, _ error) {
23+
stdoutReader, stdoutWriter, err := os.Pipe()
24+
if err != nil {
25+
return nil, err
26+
}
27+
opts := &RunOpts{
28+
Dir: repoPath,
29+
Stdout: stdoutWriter,
30+
PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error {
31+
_ = stdoutWriter.Close()
32+
defer stdoutReader.Close()
33+
34+
scanner := bufio.NewScanner(stdoutReader)
35+
for scanner.Scan() {
36+
entry, err := parseLsTreeLine(scanner.Bytes())
37+
if err != nil {
38+
cancel()
39+
return err
40+
}
41+
if entry.EntryMode == EntryModeCommit {
42+
submoduleCommits = append(submoduleCommits, TemplateSubmoduleCommit{Path: entry.Name, Commit: entry.ID.String()})
43+
}
44+
}
45+
return scanner.Err()
46+
},
47+
}
48+
err = NewCommand(ctx, "ls-tree", "-r", "--", "HEAD").Run(opts)
49+
if err != nil {
50+
return nil, fmt.Errorf("GetTemplateSubmoduleCommits: error running git ls-tree: %v", err)
51+
}
52+
return submoduleCommits, nil
53+
}
54+
55+
// AddTemplateSubmoduleIndexes Adds the given submodules to the git index.
56+
// It is only for generating new repos based on existing template, requires the .gitmodules file to be already present in the work dir.
57+
func AddTemplateSubmoduleIndexes(ctx context.Context, repoPath string, submodules []TemplateSubmoduleCommit) error {
58+
for _, submodule := range submodules {
59+
cmd := NewCommand(ctx, "update-index", "--add", "--cacheinfo", "160000").AddDynamicArguments(submodule.Commit, submodule.Path)
60+
if stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}); err != nil {
61+
log.Error("Unable to add %s as submodule to repo %s: stdout %s\nError: %v", submodule.Path, repoPath, stdout, err)
62+
return err
63+
}
64+
}
65+
return nil
66+
}

Diff for: ‎modules/git/submodule_test.go

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Copyright 2024 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package git
5+
6+
import (
7+
"context"
8+
"os"
9+
"path/filepath"
10+
"testing"
11+
12+
"github.com/stretchr/testify/assert"
13+
"github.com/stretchr/testify/require"
14+
)
15+
16+
func TestGetTemplateSubmoduleCommits(t *testing.T) {
17+
testRepoPath := filepath.Join(testReposDir, "repo4_submodules")
18+
submodules, err := GetTemplateSubmoduleCommits(DefaultContext, testRepoPath)
19+
require.NoError(t, err)
20+
21+
assert.Len(t, submodules, 2)
22+
23+
assert.EqualValues(t, "<°)))><", submodules[0].Path)
24+
assert.EqualValues(t, "d2932de67963f23d43e1c7ecf20173e92ee6c43c", submodules[0].Commit)
25+
26+
assert.EqualValues(t, "libtest", submodules[1].Path)
27+
assert.EqualValues(t, "1234567890123456789012345678901234567890", submodules[1].Commit)
28+
}
29+
30+
func TestAddTemplateSubmoduleIndexes(t *testing.T) {
31+
ctx := context.Background()
32+
tmpDir := t.TempDir()
33+
var err error
34+
_, _, err = NewCommand(ctx, "init").RunStdString(&RunOpts{Dir: tmpDir})
35+
require.NoError(t, err)
36+
_ = os.Mkdir(filepath.Join(tmpDir, "new-dir"), 0o755)
37+
err = AddTemplateSubmoduleIndexes(ctx, tmpDir, []TemplateSubmoduleCommit{{Path: "new-dir", Commit: "1234567890123456789012345678901234567890"}})
38+
require.NoError(t, err)
39+
_, _, err = NewCommand(ctx, "add", "--all").RunStdString(&RunOpts{Dir: tmpDir})
40+
require.NoError(t, err)
41+
_, _, err = NewCommand(ctx, "-c", "user.name=a", "-c", "user.email=b", "commit", "-m=test").RunStdString(&RunOpts{Dir: tmpDir})
42+
require.NoError(t, err)
43+
submodules, err := GetTemplateSubmoduleCommits(DefaultContext, tmpDir)
44+
require.NoError(t, err)
45+
assert.Len(t, submodules, 1)
46+
assert.EqualValues(t, "new-dir", submodules[0].Path)
47+
assert.EqualValues(t, "1234567890123456789012345678901234567890", submodules[0].Commit)
48+
}

Diff for: ‎modules/git/tests/repos/repo4_submodules/HEAD

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ref: refs/heads/master

Diff for: ‎modules/git/tests/repos/repo4_submodules/config

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[core]
2+
repositoryformatversion = 0
3+
filemode = true
4+
bare = true

Diff for: ‎modules/git/tests/repos/repo4_submodules/objects/97/c3d30df0e6492348292600920a6482feaebb74

110 Bytes
Binary file not shown.

Diff for: ‎modules/git/tests/repos/repo4_submodules/objects/c7/e064ed49b44523cba8a5dfbc37d2ce1bb41d34

112 Bytes
Binary file not shown.

Diff for: ‎modules/git/tests/repos/repo4_submodules/objects/e1/e59caba97193d48862d6809912043871f37437

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
x��[
2+
�0E��*�_��$M�5tifBk Iŕ�7�k~��9ܘ��ܠ���.j�� �O� ��"z�`�#I�irF��͹��$%����|4)��?t��=��:K��#[$D����^�����Ӓy�HU/�f?G

Diff for: ‎modules/git/tests/repos/repo4_submodules/refs/heads/master

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
e1e59caba97193d48862d6809912043871f37437

Diff for: ‎modules/git/tree.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ func NewTree(repo *Repository, id ObjectID) *Tree {
1717
}
1818
}
1919

20-
// SubTree get a sub tree by the sub dir path
20+
// SubTree get a subtree by the sub dir path
2121
func (t *Tree) SubTree(rpath string) (*Tree, error) {
2222
if len(rpath) == 0 {
2323
return t, nil
@@ -63,7 +63,7 @@ func (repo *Repository) LsTree(ref string, filenames ...string) ([]string, error
6363
return filelist, err
6464
}
6565

66-
// GetTreePathLatestCommitID returns the latest commit of a tree path
66+
// GetTreePathLatestCommit returns the latest commit of a tree path
6767
func (repo *Repository) GetTreePathLatestCommit(refName, treePath string) (*Commit, error) {
6868
stdout, _, err := NewCommand(repo.Ctx, "rev-list", "-1").
6969
AddDynamicArguments(refName).AddDashesAndList(treePath).

Diff for: ‎modules/git/tree_blob_nogogit.go

-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) {
1717
ptree: t,
1818
ID: t.ID,
1919
name: "",
20-
fullName: "",
2120
entryMode: EntryModeTree,
2221
}, nil
2322
}

Diff for: ‎modules/git/tree_entry_nogogit.go

+3-9
Original file line numberDiff line numberDiff line change
@@ -9,23 +9,17 @@ import "code.gitea.io/gitea/modules/log"
99

1010
// TreeEntry the leaf in the git tree
1111
type TreeEntry struct {
12-
ID ObjectID
13-
12+
ID ObjectID
1413
ptree *Tree
1514

1615
entryMode EntryMode
1716
name string
18-
19-
size int64
20-
sized bool
21-
fullName string
17+
size int64
18+
sized bool
2219
}
2320

2421
// Name returns the name of the entry
2522
func (te *TreeEntry) Name() string {
26-
if te.fullName != "" {
27-
return te.fullName
28-
}
2923
return te.name
3024
}
3125

0 commit comments

Comments
 (0)