diff --git a/models/git/branch.go b/models/git/branch.go index 9ac6c45578f7c..616f97790770c 100644 --- a/models/git/branch.go +++ b/models/git/branch.go @@ -109,6 +109,8 @@ type Branch struct { Repo *repo_model.Repository `xorm:"-"` Name string `xorm:"UNIQUE(s) NOT NULL"` // git's ref-name is case-sensitive internally, however, in some databases (mssql, mysql, by default), it's case-insensitive at the moment CommitID string + CommitCountID string // the commit id of the commit count + CommitCount int64 // the number of commits in this branch CommitMessage string `xorm:"TEXT"` // it only stores the message summary (the first line) PusherID int64 Pusher *user_model.User `xorm:"-"` @@ -263,6 +265,16 @@ func UpdateBranch(ctx context.Context, repoID, pusherID int64, branchName string }) } +func UpdateBranchCommitCount(ctx context.Context, repoID int64, branchName, commitID string, commitCount int64) error { + _, err := db.GetEngine(ctx).Where("repo_id=? AND name=?", repoID, branchName). + Cols("commit_count", "commit_count_id"). + Update(&Branch{ + CommitCount: commitCount, + CommitCountID: commitID, + }) + return err +} + // AddDeletedBranch adds a deleted branch to the database func AddDeletedBranch(ctx context.Context, repoID int64, branchName string, deletedByID int64) error { branch, err := GetBranch(ctx, repoID, branchName) diff --git a/models/git/branch_list.go b/models/git/branch_list.go index 25e84526d29e4..11b942e80dbce 100644 --- a/models/git/branch_list.go +++ b/models/git/branch_list.go @@ -146,3 +146,38 @@ func FindBranchesByRepoAndBranchName(ctx context.Context, repoBranches map[int64 } return branchMap, nil } + +func FindCommitsCountOutdatedBranches(ctx context.Context, startID, limit int64) (BranchList, error) { + var branches BranchList + if err := db.GetEngine(ctx). + Join("INNER", "repository", "branch.repo_id = repository.id"). + And("repository.is_empty = ?", false). + Where("id > ?", startID). + And( + builder.Expr("commit_count_id IS NULL"). + Or(builder.Expr("commit_id <> commit_count_id")), + ). + Asc("id"). + Limit(int(limit)).Find(&branches); err != nil { + return nil, err + } + return branches, nil +} + +func FindRepoCommitsCountOutdatedBranches(ctx context.Context, repoID, startID, limit int64) (BranchList, error) { + var branches BranchList + if err := db.GetEngine(ctx). + Join("INNER", "repository", "branch.repo_id = repository.id"). + Where("branch.repo_id = ?", repoID). + And("repository.is_empty = ?", false). + And("id > ?", startID). + And( + builder.Expr("commit_count_id IS NULL"). + Or(builder.Expr("commit_id <> commit_count_id")), + ). + Asc("id"). + Limit(int(limit)).Find(&branches); err != nil { + return nil, err + } + return branches, nil +} diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 6a60067782cff..8df4bfdd2aa64 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -380,6 +380,7 @@ func prepareMigrationTasks() []*migration { newMigration(316, "Add description for secrets and variables", v1_24.AddDescriptionForSecretsAndVariables), newMigration(317, "Add new index for action for heatmap", v1_24.AddNewIndexForUserDashboard), newMigration(318, "Add anonymous_access_mode for repo_unit", v1_24.AddRepoUnitAnonymousAccessMode), + newMigration(319, "Add branch commits count for branch table", v1_24.AddBranchCommitsCount), } return preparedMigrations } diff --git a/models/migrations/v1_24/v319.go b/models/migrations/v1_24/v319.go new file mode 100644 index 0000000000000..262018b9caecf --- /dev/null +++ b/models/migrations/v1_24/v319.go @@ -0,0 +1,16 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_24 //nolint + +import ( + "xorm.io/xorm" +) + +func AddBranchCommitsCount(x *xorm.Engine) error { + type Branch struct { + CommitCountID string // the commit id of the commit count + CommitCount int64 // the number of commits in this branch + } + return x.Sync(new(Branch)) +} diff --git a/modules/git/ref.go b/modules/git/ref.go index f20a175e422a8..b8b3dbd5b5aee 100644 --- a/modules/git/ref.go +++ b/modules/git/ref.go @@ -84,6 +84,18 @@ func RefNameFromCommit(shortName string) RefName { return RefName(shortName) } +func RefNameFromObjectTypeAndShortName(objectType ObjectType, shortName string) RefName { + switch objectType { + case ObjectBranch: + return RefNameFromBranch(shortName) + case ObjectTag: + return RefNameFromTag(shortName) + case ObjectCommit: + return RefNameFromCommit(shortName) + } + return RefName(shortName) +} + func (ref RefName) String() string { return string(ref) } diff --git a/modules/git/repo_commit.go b/modules/git/repo_commit.go index 72f35711f0fd6..01067f4501881 100644 --- a/modules/git/repo_commit.go +++ b/modules/git/repo_commit.go @@ -502,18 +502,8 @@ func (repo *Repository) IsCommitInBranch(commitID, branch string) (r bool, err e return len(stdout) > 0, err } -func (repo *Repository) AddLastCommitCache(cacheKey, fullName, sha string) error { +func (repo *Repository) AddLastCommitCache(commitsCount int64, fullName, sha string) error { if repo.LastCommitCache == nil { - commitsCount, err := cache.GetInt64(cacheKey, func() (int64, error) { - commit, err := repo.GetCommit(sha) - if err != nil { - return 0, err - } - return commit.CommitsCount() - }) - if err != nil { - return err - } repo.LastCommitCache = NewLastCommitCache(commitsCount, fullName, repo, cache.GetCache()) } return nil diff --git a/modules/gitrepo/commit.go b/modules/gitrepo/commit.go new file mode 100644 index 0000000000000..344c3f6051db0 --- /dev/null +++ b/modules/gitrepo/commit.go @@ -0,0 +1,17 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitrepo + +import ( + "context" + + "code.gitea.io/gitea/modules/git" +) + +func CommitsCount(ctx context.Context, repo Repository, ref string) (int64, error) { + return git.CommitsCount(ctx, git.CommitsCountOptions{ + RepoPath: repoPath(repo), + Revision: []string{ref}, + }) +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 96c99615f5d52..7068e1f90e2b3 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3026,6 +3026,7 @@ dashboard.sync_branch.started = Branches Sync started dashboard.sync_tag.started = Tags Sync started dashboard.rebuild_issue_indexer = Rebuild issue indexer dashboard.sync_repo_licenses = Sync repo licenses +dashboard.sync_branch_commits_count = Sync Branch commits counts users.user_manage_panel = User Account Management users.new_account = Create User Account diff --git a/routers/api/v1/utils/git.go b/routers/api/v1/utils/git.go index 7d276d9d98bae..c8cc9fe055121 100644 --- a/routers/api/v1/utils/git.go +++ b/routers/api/v1/utils/git.go @@ -39,7 +39,8 @@ func ResolveRefOrSha(ctx *context.APIContext, ref string) string { sha = MustConvertToSHA1(ctx, ctx.Repo, sha) if ctx.Repo.GitRepo != nil { - err := ctx.Repo.GitRepo.AddLastCommitCache(ctx.Repo.Repository.GetCommitsCountCacheKey(ref, ref != sha), ctx.Repo.Repository.FullName(), sha) + commitsCount, _ := context.GetRefCommitsCount(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, git.RefName(ref)) + err := ctx.Repo.GitRepo.AddLastCommitCache(commitsCount, ctx.Repo.Repository.FullName(), sha) if err != nil { log.Error("Unable to get commits count for %s in %s. Error: %v", sha, ctx.Repo.Repository.FullName(), err) } diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index e260ea36ddd76..de77546155933 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -71,7 +71,7 @@ func CommitInfoCache(ctx *context.Context) { ctx.ServerError("GetBranchCommit", err) return } - ctx.Repo.CommitsCount, err = ctx.Repo.GetCommitsCount() + ctx.Repo.CommitsCount, err = ctx.Repo.GetCommitsCount(ctx) if err != nil { ctx.ServerError("GetCommitsCount", err) return diff --git a/services/context/repo.go b/services/context/repo.go index 4e91e53e7d083..15f23adc9dfdd 100644 --- a/services/context/repo.go +++ b/services/context/repo.go @@ -163,16 +163,45 @@ func (r *Repository) CanCreateIssueDependencies(ctx context.Context, user *user_ return r.Repository.IsDependenciesEnabled(ctx) && r.Permission.CanWriteIssuesOrPulls(isPull) } +// getCommitsCountCacheKey returns cache key used for commits count caching. +func getCommitsCountCacheKey(contextName string, repoID int64) string { + return fmt.Sprintf("commits-count-%d-commit-%s", repoID, contextName) +} + +func GetRefCommitsCount(ctx context.Context, repoID int64, gitRepo *git.Repository, refFullName git.RefName) (int64, error) { + // Get the commit count of the branch or the tag + switch { + case refFullName.IsBranch(): + branch, err := git_model.GetBranch(ctx, repoID, refFullName.BranchName()) + if err != nil { + return 0, err + } + return branch.CommitCount, nil + case refFullName.IsTag(): + tag, err := repo_model.GetRelease(ctx, repoID, refFullName.TagName()) + if err != nil { + return 0, err + } + return tag.NumCommits, nil + case refFullName.RefType() == git.RefTypeCommit: + return cache.GetInt64(getCommitsCountCacheKey(string(refFullName), repoID), func() (int64, error) { + commit, err := gitRepo.GetCommit(string(refFullName)) + if err != nil { + return 0, err + } + return commit.CommitsCount() + }) + default: + return 0, nil + } +} + // GetCommitsCount returns cached commit count for current view -func (r *Repository) GetCommitsCount() (int64, error) { +func (r *Repository) GetCommitsCount(ctx context.Context) (int64, error) { if r.Commit == nil { return 0, nil } - contextName := r.RefFullName.ShortName() - isRef := r.RefFullName.IsBranch() || r.RefFullName.IsTag() - return cache.GetInt64(r.Repository.GetCommitsCountCacheKey(contextName, isRef), func() (int64, error) { - return r.Commit.CommitsCount() - }) + return GetRefCommitsCount(ctx, r.Repository.ID, r.GitRepo, r.RefFullName) } // GetCommitGraphsCount returns cached commit count for current view @@ -784,7 +813,7 @@ func RepoRefByDefaultBranch() func(*Context) { ctx.Repo.RefFullName = git.RefNameFromBranch(ctx.Repo.Repository.DefaultBranch) ctx.Repo.BranchName = ctx.Repo.Repository.DefaultBranch ctx.Repo.Commit, _ = ctx.Repo.GitRepo.GetBranchCommit(ctx.Repo.BranchName) - ctx.Repo.CommitsCount, _ = ctx.Repo.GetCommitsCount() + ctx.Repo.CommitsCount, _ = ctx.Repo.GetCommitsCount(ctx) ctx.Data["RefFullName"] = ctx.Repo.RefFullName ctx.Data["BranchName"] = ctx.Repo.BranchName ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount @@ -933,7 +962,7 @@ func RepoRefByType(detectRefType git.RefType) func(*Context) { ctx.Data["CanCreateBranch"] = ctx.Repo.CanCreateBranch() // only used by the branch selector dropdown: AllowCreateNewRef - ctx.Repo.CommitsCount, err = ctx.Repo.GetCommitsCount() + ctx.Repo.CommitsCount, err = ctx.Repo.GetCommitsCount(ctx) if err != nil { ctx.ServerError("GetCommitsCount", err) return diff --git a/services/cron/tasks_basic.go b/services/cron/tasks_basic.go index 841981787dffd..8bb83d3fc7a09 100644 --- a/services/cron/tasks_basic.go +++ b/services/cron/tasks_basic.go @@ -166,6 +166,16 @@ func registerSyncRepoLicenses() { }) } +func registerSyncBranchCommitsCount() { + RegisterTaskFatal("sync_branch_commits_count", &BaseConfig{ + Enabled: true, + RunAtStart: true, + Schedule: "@midnight", + }, func(ctx context.Context, _ *user_model.User, _ Config) error { + return repo_service.SyncBranchCommitsCount(ctx) + }) +} + func initBasicTasks() { if setting.Mirror.Enabled { registerUpdateMirrorTask() @@ -183,4 +193,5 @@ func initBasicTasks() { registerCleanupPackages() } registerSyncRepoLicenses() + registerSyncBranchCommitsCount() } diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index c43a4ef04a181..b14ebe0680918 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -11,7 +11,6 @@ import ( repo_model "code.gitea.io/gitea/models/repo" system_model "code.gitea.io/gitea/models/system" - "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/git" giturl "code.gitea.io/gitea/modules/git/url" "code.gitea.io/gitea/modules/gitrepo" @@ -429,15 +428,8 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo log.Trace("SyncMirrors [repo: %-v Wiki]: git remote update complete", m.Repo) } - log.Trace("SyncMirrors [repo: %-v]: invalidating mirror branch caches...", m.Repo) - branches, _, err := gitrepo.GetBranchesByPath(ctx, m.Repo, 0, 0) - if err != nil { - log.Error("SyncMirrors [repo: %-v]: failed to GetBranches: %v", m.Repo, err) - return nil, false - } - - for _, branch := range branches { - cache.Remove(m.Repo.GetCommitsCountCacheKey(branch, true)) + if err := repo_service.SyncRepoBranchesCommitsCount(ctx, m.Repo); err != nil { + log.Error("SyncMirrors [repo: %-v]: failed to sync branches commits count: %v", m.Repo, err) } m.UpdatedUnix = timeutil.TimeStampNow() diff --git a/services/pull/merge.go b/services/pull/merge.go index 9804d8aac1ed6..9ceabda7b5d01 100644 --- a/services/pull/merge.go +++ b/services/pull/merge.go @@ -22,7 +22,6 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/globallock" "code.gitea.io/gitea/modules/httplib" @@ -252,9 +251,6 @@ func Merge(ctx context.Context, pr *issues_model.PullRequest, doer *user_model.U notify_service.MergePullRequest(ctx, doer, pr) } - // Reset cached commit count - cache.Remove(pr.Issue.Repo.GetCommitsCountCacheKey(pr.BaseBranch, true)) - return handleCloseCrossReferences(ctx, pr, doer) } diff --git a/services/repository/branch.go b/services/repository/branch.go index 94c47ffdc47c4..331f05d925d2b 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -736,3 +736,79 @@ func GetBranchDivergingInfo(ctx reqctx.RequestContext, baseRepo *repo_model.Repo info.BaseHasNewCommits = info.HeadCommitsBehind > 0 return info, nil } + +func SyncRepoBranchesCommitsCount(ctx context.Context, repo *repo_model.Repository) error { + startID := int64(0) + for { + select { + case <-ctx.Done(): + return nil + default: + } + + // search all branches commits count are not synced + branches, err := git_model.FindRepoCommitsCountOutdatedBranches(ctx, repo.ID, startID, 100) + if err != nil { + return err + } + if len(branches) == 0 { + return nil + } + + for _, branch := range branches { + branch.Repo = repo + if branch.ID > startID { + startID = branch.ID + } + if err := syncBranchCommitsCount(ctx, branch); err != nil { + log.Error("syncBranchCommitsCount: %v", err) + } + } + } +} + +func SyncBranchCommitsCount(ctx context.Context) error { + startID := int64(0) + for { + select { + case <-ctx.Done(): + return nil + default: + } + + // search all branches commits count are not synced + branches, err := git_model.FindCommitsCountOutdatedBranches(ctx, startID, 100) + if err != nil { + return err + } + if len(branches) == 0 { + return nil + } + + if err := branches.LoadRepo(ctx); err != nil { + return err + } + + for _, branch := range branches { + if branch.ID > startID { + startID = branch.ID + } + if err := syncBranchCommitsCount(ctx, branch); err != nil { + log.Error("syncBranchCommitsCount: %v", err) + } + } + } +} + +func syncBranchCommitsCount(ctx context.Context, branch *git_model.Branch) error { + if branch.CommitID == "" { + return nil + } + + commitsCount, err := gitrepo.CommitsCount(ctx, branch.Repo, branch.CommitID) + if err != nil { + return err + } + + return git_model.UpdateBranchCommitCount(ctx, branch.RepoID, branch.Name, branch.CommitID, commitsCount) +} diff --git a/services/repository/cache.go b/services/repository/cache.go index b0811a99fc03b..648ebc4c97e45 100644 --- a/services/repository/cache.go +++ b/services/repository/cache.go @@ -9,6 +9,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/git" + context_service "code.gitea.io/gitea/services/context" ) // CacheRef cachhe last commit information of the branch or the tag @@ -19,7 +20,7 @@ func CacheRef(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Rep } if gitRepo.LastCommitCache == nil { - commitsCount, err := cache.GetInt64(repo.GetCommitsCountCacheKey(fullRefName.ShortName(), true), commit.CommitsCount) + commitsCount, err := context_service.GetRefCommitsCount(ctx, repo.ID, gitRepo, fullRefName) if err != nil { return err } diff --git a/services/repository/files/content.go b/services/repository/files/content.go index 0ab7422ce2ebf..d223dbb5fe4c0 100644 --- a/services/repository/files/content.go +++ b/services/repository/files/content.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" + context_service "code.gitea.io/gitea/services/context" ) // ContentType repo content type @@ -165,7 +166,11 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref } selfURLString := selfURL.String() - err = gitRepo.AddLastCommitCache(repo.GetCommitsCountCacheKey(ref, refType != git.ObjectCommit), repo.FullName(), commitID) + refName := git.RefNameFromObjectTypeAndShortName(refType, ref) + + commitsCount, _ := context_service.GetRefCommitsCount(ctx, repo.ID, gitRepo, refName) + + err = gitRepo.AddLastCommitCache(commitsCount, repo.FullName(), commitID) if err != nil { return nil, err } diff --git a/services/repository/push.go b/services/repository/push.go index 7d4e24188d5e2..13969257434e0 100644 --- a/services/repository/push.go +++ b/services/repository/push.go @@ -11,9 +11,9 @@ import ( "time" "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/graceful" @@ -286,6 +286,11 @@ func pushNewBranch(ctx context.Context, repo *repo_model.Repository, pusher *use if err := repo_model.UpdateRepositoryCols(ctx, repo, "default_branch", "is_empty"); err != nil { return nil, fmt.Errorf("UpdateRepositoryCols: %w", err) } + } else { + // calculate the number of commits in the branch + if err := UpdateRepoBranchCommitsCount(ctx, repo, opts.RefFullName.BranchName(), newCommit); err != nil { + log.Error("UpdateRepoBranchCommitsCount: %v", err) + } } l, err := newCommit.CommitsBeforeLimit(10) @@ -296,7 +301,16 @@ func pushNewBranch(ctx context.Context, repo *repo_model.Repository, pusher *use return l, nil } -func pushUpdateBranch(_ context.Context, repo *repo_model.Repository, pusher *user_model.User, opts *repo_module.PushUpdateOptions, newCommit *git.Commit) ([]*git.Commit, error) { +func UpdateRepoBranchCommitsCount(ctx context.Context, repo *repo_model.Repository, branch string, newCommit *git.Commit) error { + // calculate the number of commits in the branch + commitsCount, err := newCommit.CommitsCount() + if err != nil { + return fmt.Errorf("newCommit.CommitsCount: %w", err) + } + return git_model.UpdateBranchCommitCount(ctx, repo.ID, branch, newCommit.ID.String(), commitsCount) +} + +func pushUpdateBranch(ctx context.Context, repo *repo_model.Repository, pusher *user_model.User, opts *repo_module.PushUpdateOptions, newCommit *git.Commit) ([]*git.Commit, error) { l, err := newCommit.CommitsBeforeUntil(opts.OldCommitID) if err != nil { return nil, fmt.Errorf("newCommit.CommitsBeforeUntil: %w", err) @@ -304,6 +318,11 @@ func pushUpdateBranch(_ context.Context, repo *repo_model.Repository, pusher *us branch := opts.RefFullName.BranchName() + // calculate the number of commits in the branch + if err := UpdateRepoBranchCommitsCount(ctx, repo, branch, newCommit); err != nil { + log.Error("UpdateRepoBranchCommitsCount: %v", err) + } + isForcePush, err := newCommit.IsForcePush(opts.OldCommitID) if err != nil { log.Error("IsForcePush %s:%s failed: %v", repo.FullName(), branch, err) @@ -320,15 +339,6 @@ func pushUpdateBranch(_ context.Context, repo *repo_model.Repository, pusher *us NewCommitID: opts.NewCommitID, }) - if isForcePush { - log.Trace("Push %s is a force push", opts.NewCommitID) - - cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true)) - } else { - // TODO: increment update the commit count cache but not remove - cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true)) - } - return l, nil }