Skip to content

Commit 3927178

Browse files
zjjhotzjjhot
committed
Merge remote-tracking branch 'giteaofficial/main'
* giteaofficial/main: Add new index for action to resolve the performance problem (go-gitea#32333) Include file extension checks in attachment API (go-gitea#32151) Updated tokenizer to better matching when search for code snippets (go-gitea#32261) Correctly query the primary button in a form (go-gitea#32438) # Conflicts: # web_src/js/utils/dom.ts
2 parents 952637f + 913be9e commit 3927178

30 files changed

+289
-31
lines changed

‎models/activities/action.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,10 @@ func (a *Action) TableIndices() []*schemas.Index {
171171
cudIndex := schemas.NewIndex("c_u_d", schemas.IndexType)
172172
cudIndex.AddColumn("created_unix", "user_id", "is_deleted")
173173

174-
indices := []*schemas.Index{actUserIndex, repoIndex, cudIndex}
174+
cuIndex := schemas.NewIndex("c_u", schemas.IndexType)
175+
cuIndex.AddColumn("user_id", "is_deleted")
176+
177+
indices := []*schemas.Index{actUserIndex, repoIndex, cudIndex, cuIndex}
175178

176179
return indices
177180
}

‎models/migrations/migrations.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -365,6 +365,7 @@ func prepareMigrationTasks() []*migration {
365365
newMigration(305, "Add Repository Licenses", v1_23.AddRepositoryLicenses),
366366
newMigration(306, "Add BlockAdminMergeOverride to ProtectedBranch", v1_23.AddBlockAdminMergeOverrideBranchProtection),
367367
newMigration(307, "Fix milestone deadline_unix when there is no due date", v1_23.FixMilestoneNoDueDate),
368+
newMigration(308, "Add index(user_id, is_deleted) for action table", v1_23.AddNewIndexForUserDashboard),
368369
}
369370
return preparedMigrations
370371
}

‎models/migrations/v1_23/v308.go

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
// Copyright 2024 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package v1_23 //nolint
5+
6+
import (
7+
"code.gitea.io/gitea/modules/timeutil"
8+
9+
"xorm.io/xorm"
10+
"xorm.io/xorm/schemas"
11+
)
12+
13+
type improveActionTableIndicesAction struct {
14+
ID int64 `xorm:"pk autoincr"`
15+
UserID int64 `xorm:"INDEX"` // Receiver user id.
16+
OpType int
17+
ActUserID int64 // Action user id.
18+
RepoID int64
19+
CommentID int64 `xorm:"INDEX"`
20+
IsDeleted bool `xorm:"NOT NULL DEFAULT false"`
21+
RefName string
22+
IsPrivate bool `xorm:"NOT NULL DEFAULT false"`
23+
Content string `xorm:"TEXT"`
24+
CreatedUnix timeutil.TimeStamp `xorm:"created"`
25+
}
26+
27+
// TableName sets the name of this table
28+
func (*improveActionTableIndicesAction) TableName() string {
29+
return "action"
30+
}
31+
32+
func (a *improveActionTableIndicesAction) TableIndices() []*schemas.Index {
33+
repoIndex := schemas.NewIndex("r_u_d", schemas.IndexType)
34+
repoIndex.AddColumn("repo_id", "user_id", "is_deleted")
35+
36+
actUserIndex := schemas.NewIndex("au_r_c_u_d", schemas.IndexType)
37+
actUserIndex.AddColumn("act_user_id", "repo_id", "created_unix", "user_id", "is_deleted")
38+
39+
cudIndex := schemas.NewIndex("c_u_d", schemas.IndexType)
40+
cudIndex.AddColumn("created_unix", "user_id", "is_deleted")
41+
42+
cuIndex := schemas.NewIndex("c_u", schemas.IndexType)
43+
cuIndex.AddColumn("user_id", "is_deleted")
44+
45+
indices := []*schemas.Index{actUserIndex, repoIndex, cudIndex, cuIndex}
46+
47+
return indices
48+
}
49+
50+
func AddNewIndexForUserDashboard(x *xorm.Engine) error {
51+
return x.Sync(new(improveActionTableIndicesAction))
52+
}

‎modules/indexer/code/bleve/bleve.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"github.com/blevesearch/bleve/v2/analysis/token/camelcase"
3232
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
3333
"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
34+
"github.com/blevesearch/bleve/v2/analysis/tokenizer/letter"
3435
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
3536
"github.com/blevesearch/bleve/v2/mapping"
3637
"github.com/blevesearch/bleve/v2/search/query"
@@ -69,7 +70,7 @@ const (
6970
filenameIndexerAnalyzer = "filenameIndexerAnalyzer"
7071
filenameIndexerTokenizer = "filenameIndexerTokenizer"
7172
repoIndexerDocType = "repoIndexerDocType"
72-
repoIndexerLatestVersion = 7
73+
repoIndexerLatestVersion = 8
7374
)
7475

7576
// generateBleveIndexMapping generates a bleve index mapping for the repo indexer
@@ -105,7 +106,7 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) {
105106
} else if err := mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]any{
106107
"type": analyzer_custom.Name,
107108
"char_filters": []string{},
108-
"tokenizer": unicode.Name,
109+
"tokenizer": letter.Name,
109110
"token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name},
110111
}); err != nil {
111112
return nil, err

‎modules/indexer/code/elasticsearch/elasticsearch.go

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import (
3030
)
3131

3232
const (
33-
esRepoIndexerLatestVersion = 2
33+
esRepoIndexerLatestVersion = 3
3434
// multi-match-types, currently only 2 types are used
3535
// Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types
3636
esMultiMatchTypeBestFields = "best_fields"
@@ -60,6 +60,10 @@ const (
6060
"settings": {
6161
"analysis": {
6262
"analyzer": {
63+
"content_analyzer": {
64+
"tokenizer": "content_tokenizer",
65+
"filter" : ["lowercase"]
66+
},
6367
"filename_path_analyzer": {
6468
"tokenizer": "path_tokenizer"
6569
},
@@ -68,6 +72,10 @@ const (
6872
}
6973
},
7074
"tokenizer": {
75+
"content_tokenizer": {
76+
"type": "simple_pattern_split",
77+
"pattern": "[^a-zA-Z0-9]"
78+
},
7179
"path_tokenizer": {
7280
"type": "path_hierarchy",
7381
"delimiter": "/"
@@ -104,7 +112,8 @@ const (
104112
"content": {
105113
"type": "text",
106114
"term_vector": "with_positions_offsets",
107-
"index": true
115+
"index": true,
116+
"analyzer": "content_analyzer"
108117
},
109118
"commit_id": {
110119
"type": "keyword",

‎modules/indexer/code/indexer_test.go

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,55 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
181181
},
182182
},
183183
},
184+
// Search for matches on the contents of files regardless of case.
185+
{
186+
RepoIDs: nil,
187+
Keyword: "dESCRIPTION",
188+
Langs: 1,
189+
Results: []codeSearchResult{
190+
{
191+
Filename: "README.md",
192+
Content: "# repo1\n\nDescription for repo1",
193+
},
194+
},
195+
},
196+
// Search for an exact match on the filename within the repo '62' (case insenstive).
197+
// This scenario yields a single result (the file avocado.md on the repo '62')
198+
{
199+
RepoIDs: []int64{62},
200+
Keyword: "AVOCADO.MD",
201+
Langs: 1,
202+
Results: []codeSearchResult{
203+
{
204+
Filename: "avocado.md",
205+
Content: "# repo1\n\npineaple pie of cucumber juice",
206+
},
207+
},
208+
},
209+
// Search for matches on the contents of files when the criteria is a expression.
210+
{
211+
RepoIDs: []int64{62},
212+
Keyword: "console.log",
213+
Langs: 1,
214+
Results: []codeSearchResult{
215+
{
216+
Filename: "example-file.js",
217+
Content: "console.log(\"Hello, World!\")",
218+
},
219+
},
220+
},
221+
// Search for matches on the contents of files when the criteria is part of a expression.
222+
{
223+
RepoIDs: []int64{62},
224+
Keyword: "log",
225+
Langs: 1,
226+
Results: []codeSearchResult{
227+
{
228+
Filename: "example-file.js",
229+
Content: "console.log(\"Hello, World!\")",
230+
},
231+
},
232+
},
184233
}
185234

186235
for _, kw := range keywords {

‎modules/indexer/internal/bleve/util.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,13 @@ package bleve
66
import (
77
"errors"
88
"os"
9+
"unicode"
910

1011
"code.gitea.io/gitea/modules/log"
1112
"code.gitea.io/gitea/modules/util"
1213

1314
"github.com/blevesearch/bleve/v2"
14-
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
15+
unicode_tokenizer "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
1516
"github.com/blevesearch/bleve/v2/index/upsidedown"
1617
"github.com/ethantkoenig/rupture"
1718
)
@@ -57,7 +58,7 @@ func openIndexer(path string, latestVersion int) (bleve.Index, int, error) {
5758
// may be different on two string and they still be considered equivalent.
5859
// Given a phrasse, its shortest word determines its fuzziness. If a phrase uses CJK (eg: `갃갃갃` `啊啊啊`), the fuzziness is zero.
5960
func GuessFuzzinessByKeyword(s string) int {
60-
tokenizer := unicode.NewUnicodeTokenizer()
61+
tokenizer := unicode_tokenizer.NewUnicodeTokenizer()
6162
tokens := tokenizer.Tokenize([]byte(s))
6263

6364
if len(tokens) > 0 {
@@ -77,8 +78,10 @@ func guessFuzzinessByKeyword(s string) int {
7778
// according to https://github.com/blevesearch/bleve/issues/1563, the supported max fuzziness is 2
7879
// magic number 4 was chosen to determine the levenshtein distance per each character of a keyword
7980
// BUT, when using CJK (eg: `갃갃갃` `啊啊啊`), it mismatches a lot.
81+
// Likewise, queries whose terms contains characters that are *not* letters should not use fuzziness
82+
8083
for _, r := range s {
81-
if r >= 128 {
84+
if r >= 128 || !unicode.IsLetter(r) {
8285
return 0
8386
}
8487
}

‎modules/indexer/internal/bleve/util_test.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,14 @@ func TestBleveGuessFuzzinessByKeyword(t *testing.T) {
3535
Input: "갃갃갃",
3636
Fuzziness: 0,
3737
},
38+
{
39+
Input: "repo1",
40+
Fuzziness: 0,
41+
},
42+
{
43+
Input: "avocado.md",
44+
Fuzziness: 0,
45+
},
3846
}
3947

4048
for _, scenario := range scenarios {

‎routers/api/v1/repo/issue_attachment.go

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import (
1212
"code.gitea.io/gitea/modules/setting"
1313
api "code.gitea.io/gitea/modules/structs"
1414
"code.gitea.io/gitea/modules/web"
15-
"code.gitea.io/gitea/services/attachment"
15+
attachment_service "code.gitea.io/gitea/services/attachment"
1616
"code.gitea.io/gitea/services/context"
1717
"code.gitea.io/gitea/services/context/upload"
1818
"code.gitea.io/gitea/services/convert"
@@ -181,7 +181,7 @@ func CreateIssueAttachment(ctx *context.APIContext) {
181181
filename = query
182182
}
183183

184-
attachment, err := attachment.UploadAttachment(ctx, file, setting.Attachment.AllowedTypes, header.Size, &repo_model.Attachment{
184+
attachment, err := attachment_service.UploadAttachment(ctx, file, setting.Attachment.AllowedTypes, header.Size, &repo_model.Attachment{
185185
Name: filename,
186186
UploaderID: ctx.Doer.ID,
187187
RepoID: ctx.Repo.Repository.ID,
@@ -247,6 +247,8 @@ func EditIssueAttachment(ctx *context.APIContext) {
247247
// "$ref": "#/responses/Attachment"
248248
// "404":
249249
// "$ref": "#/responses/error"
250+
// "422":
251+
// "$ref": "#/responses/validationError"
250252
// "423":
251253
// "$ref": "#/responses/repoArchivedError"
252254

@@ -261,8 +263,13 @@ func EditIssueAttachment(ctx *context.APIContext) {
261263
attachment.Name = form.Name
262264
}
263265

264-
if err := repo_model.UpdateAttachment(ctx, attachment); err != nil {
266+
if err := attachment_service.UpdateAttachment(ctx, setting.Attachment.AllowedTypes, attachment); err != nil {
267+
if upload.IsErrFileTypeForbidden(err) {
268+
ctx.Error(http.StatusUnprocessableEntity, "", err)
269+
return
270+
}
265271
ctx.Error(http.StatusInternalServerError, "UpdateAttachment", err)
272+
return
266273
}
267274

268275
ctx.JSON(http.StatusCreated, convert.ToAPIAttachment(ctx.Repo.Repository, attachment))

‎routers/api/v1/repo/issue_comment_attachment.go

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import (
1414
"code.gitea.io/gitea/modules/setting"
1515
api "code.gitea.io/gitea/modules/structs"
1616
"code.gitea.io/gitea/modules/web"
17-
"code.gitea.io/gitea/services/attachment"
17+
attachment_service "code.gitea.io/gitea/services/attachment"
1818
"code.gitea.io/gitea/services/context"
1919
"code.gitea.io/gitea/services/context/upload"
2020
"code.gitea.io/gitea/services/convert"
@@ -189,7 +189,7 @@ func CreateIssueCommentAttachment(ctx *context.APIContext) {
189189
filename = query
190190
}
191191

192-
attachment, err := attachment.UploadAttachment(ctx, file, setting.Attachment.AllowedTypes, header.Size, &repo_model.Attachment{
192+
attachment, err := attachment_service.UploadAttachment(ctx, file, setting.Attachment.AllowedTypes, header.Size, &repo_model.Attachment{
193193
Name: filename,
194194
UploaderID: ctx.Doer.ID,
195195
RepoID: ctx.Repo.Repository.ID,
@@ -263,6 +263,8 @@ func EditIssueCommentAttachment(ctx *context.APIContext) {
263263
// "$ref": "#/responses/Attachment"
264264
// "404":
265265
// "$ref": "#/responses/error"
266+
// "422":
267+
// "$ref": "#/responses/validationError"
266268
// "423":
267269
// "$ref": "#/responses/repoArchivedError"
268270
attach := getIssueCommentAttachmentSafeWrite(ctx)
@@ -275,8 +277,13 @@ func EditIssueCommentAttachment(ctx *context.APIContext) {
275277
attach.Name = form.Name
276278
}
277279

278-
if err := repo_model.UpdateAttachment(ctx, attach); err != nil {
280+
if err := attachment_service.UpdateAttachment(ctx, setting.Attachment.AllowedTypes, attach); err != nil {
281+
if upload.IsErrFileTypeForbidden(err) {
282+
ctx.Error(http.StatusUnprocessableEntity, "", err)
283+
return
284+
}
279285
ctx.Error(http.StatusInternalServerError, "UpdateAttachment", attach)
286+
return
280287
}
281288
ctx.JSON(http.StatusCreated, convert.ToAPIAttachment(ctx.Repo.Repository, attach))
282289
}

‎routers/api/v1/repo/release_attachment.go

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import (
1313
"code.gitea.io/gitea/modules/setting"
1414
api "code.gitea.io/gitea/modules/structs"
1515
"code.gitea.io/gitea/modules/web"
16-
"code.gitea.io/gitea/services/attachment"
16+
attachment_service "code.gitea.io/gitea/services/attachment"
1717
"code.gitea.io/gitea/services/context"
1818
"code.gitea.io/gitea/services/context/upload"
1919
"code.gitea.io/gitea/services/convert"
@@ -234,7 +234,7 @@ func CreateReleaseAttachment(ctx *context.APIContext) {
234234
}
235235

236236
// Create a new attachment and save the file
237-
attach, err := attachment.UploadAttachment(ctx, content, setting.Repository.Release.AllowedTypes, size, &repo_model.Attachment{
237+
attach, err := attachment_service.UploadAttachment(ctx, content, setting.Repository.Release.AllowedTypes, size, &repo_model.Attachment{
238238
Name: filename,
239239
UploaderID: ctx.Doer.ID,
240240
RepoID: ctx.Repo.Repository.ID,
@@ -291,6 +291,8 @@ func EditReleaseAttachment(ctx *context.APIContext) {
291291
// responses:
292292
// "201":
293293
// "$ref": "#/responses/Attachment"
294+
// "422":
295+
// "$ref": "#/responses/validationError"
294296
// "404":
295297
// "$ref": "#/responses/notFound"
296298

@@ -322,8 +324,13 @@ func EditReleaseAttachment(ctx *context.APIContext) {
322324
attach.Name = form.Name
323325
}
324326

325-
if err := repo_model.UpdateAttachment(ctx, attach); err != nil {
327+
if err := attachment_service.UpdateAttachment(ctx, setting.Repository.Release.AllowedTypes, attach); err != nil {
328+
if upload.IsErrFileTypeForbidden(err) {
329+
ctx.Error(http.StatusUnprocessableEntity, "", err)
330+
return
331+
}
326332
ctx.Error(http.StatusInternalServerError, "UpdateAttachment", attach)
333+
return
327334
}
328335
ctx.JSON(http.StatusCreated, convert.ToAPIAttachment(ctx.Repo.Repository, attach))
329336
}

‎services/attachment/attachment.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,3 +50,12 @@ func UploadAttachment(ctx context.Context, file io.Reader, allowedTypes string,
5050

5151
return NewAttachment(ctx, attach, io.MultiReader(bytes.NewReader(buf), file), fileSize)
5252
}
53+
54+
// UpdateAttachment updates an attachment, verifying that its name is among the allowed types.
55+
func UpdateAttachment(ctx context.Context, allowedTypes string, attach *repo_model.Attachment) error {
56+
if err := upload.Verify(nil, attach.Name, allowedTypes); err != nil {
57+
return err
58+
}
59+
60+
return repo_model.UpdateAttachment(ctx, attach)
61+
}

0 commit comments

Comments
 (0)