This is an automated email from the ASF dual-hosted git repository. klesh pushed a commit to branch feat/gitextractor-git-cli in repository https://gitbox.apache.org/repos/asf/devlake.git
commit e7c816e0a3a074aaa554d086f28f8b0dcf544660 Author: Klesh Wong <[email protected]> AuthorDate: Sun Nov 1 18:48:42 2026 +0800 perf(gitextractor): only write component rows when components exist Nothing in DevLake populates the `components` table, so the component map is empty for normal repositories and every commit file got a "Default" row in `commit_file_components`. That is pure write amplification: 67,520 rows for the 8,383 commits of apache/devlake. Skip the component rows entirely when the repository has no components configured, and skip the component query when commit files are not collected. When components are configured, keep writing the correct per-file rows. The previous libgit2 collector only wrote a row for the last file of each commit, which was not a documented optimisation and assigns an arbitrary file's component; go-git already wrote one row per file. --- .../gitextractor/parser/collector_contract_test.go | 7 ++--- backend/plugins/gitextractor/parser/repo_gitcli.go | 36 +++++++++++++--------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/backend/plugins/gitextractor/parser/collector_contract_test.go b/backend/plugins/gitextractor/parser/collector_contract_test.go index 6e2bf8d07..c2029961b 100644 --- a/backend/plugins/gitextractor/parser/collector_contract_test.go +++ b/backend/plugins/gitextractor/parser/collector_contract_test.go @@ -337,10 +337,9 @@ func TestCollectorCommits(t *testing.T) { for _, file := range store.commitFiles { assert.Equal(t, genCommitFileId(file.CommitSha, file.FilePath), file.Id) } - components := componentsById(store.components) - for _, file := range store.commitFiles { - assert.Equal(t, "Default", components[file.Id], file.FilePath) - } + // This repository has no components configured, so every file would + // only get a meaningless "Default" component row. + assert.Empty(t, store.components) }) } diff --git a/backend/plugins/gitextractor/parser/repo_gitcli.go b/backend/plugins/gitextractor/parser/repo_gitcli.go index f13256daa..60ef6b709 100644 --- a/backend/plugins/gitextractor/parser/repo_gitcli.go +++ b/backend/plugins/gitextractor/parser/repo_gitcli.go @@ -230,9 +230,12 @@ func (r *GitcliRepoCollector) CollectCommits(subtaskCtx plugin.SubTaskContext) ( if err != nil { return err } - componentMap, err := r.componentMap(subtaskCtx) - if err != nil { - return err + var componentMap map[string]*regexp.Regexp + if !*taskOpts.SkipCommitFiles { + componentMap, err = r.componentMap(subtaskCtx) + if err != nil { + return err + } } excluded := normalizeExtensions(taskOpts.ExcludeFileExtensions) @@ -344,6 +347,9 @@ func (r *GitcliRepoCollector) storeParentCommits(commit *parsedCommit, present m return r.store.CommitParents(parents) } +// storeCommitFile stores one changed file of a commit. Component rows are only +// written when the repository has components configured: without them every row +// would be "Default" for every changed file, which is pure write amplification. func (r *GitcliRepoCollector) storeCommitFile(commitSha string, file commitFileStat, componentMap map[string]*regexp.Regexp) errors.Error { commitFile := &code.CommitFile{ CommitSha: commitSha, @@ -353,18 +359,20 @@ func (r *GitcliRepoCollector) storeCommitFile(commitSha string, file commitFileS } commitFile.Id = genCommitFileId(commitFile.CommitSha, commitFile.FilePath) - componentName := "Default" - for _, name := range sortedComponentNames(componentMap) { - if componentMap[name].MatchString(commitFile.FilePath) { - componentName = name - break + if len(componentMap) > 0 { + componentName := "Default" + for _, name := range sortedComponentNames(componentMap) { + if componentMap[name].MatchString(commitFile.FilePath) { + componentName = name + break + } + } + if err := r.store.CommitFileComponents(&code.CommitFileComponent{ + CommitFileId: commitFile.Id, + ComponentName: componentName, + }); err != nil { + return err } - } - if err := r.store.CommitFileComponents(&code.CommitFileComponent{ - CommitFileId: commitFile.Id, - ComponentName: componentName, - }); err != nil { - return err } return r.store.CommitFiles(commitFile) }
