This is an automated email from the ASF dual-hosted git repository. klesh pushed a commit to branch feat/gitextractor-git-cli in repository https://gitbox.apache.org/repos/asf/devlake.git
commit 082b8653ade9a0c3968575ca96d40bdc0c0c8c3b Author: Klesh Wong <[email protected]> AuthorDate: Sun Nov 1 12:37:07 2026 +0800 test(gitextractor): cover collectors with git CLI fixtures Build repository fixtures with the git command line instead of go-git and add a contract suite that records everything a RepoCollector writes: counts, refs, commits, parents, commit files, components, snapshots and line changes. The suite runs against every implementation. Behaviour that deviates from the intended contract is listed per implementation, so the git CLI collector that replaces libgit2 and go-git can be held to the contract without inheriting the deviations. No production code changes. --- .../gitextractor/parser/collector_contract_test.go | 737 +++++++++++++++++++++ .../plugins/gitextractor/parser/repo_gogit_test.go | 122 ---- .../plugins/gitextractor/parser/testrepo_test.go | 316 +++++++++ 3 files changed, 1053 insertions(+), 122 deletions(-) diff --git a/backend/plugins/gitextractor/parser/collector_contract_test.go b/backend/plugins/gitextractor/parser/collector_contract_test.go new file mode 100644 index 000000000..8ec51598a --- /dev/null +++ b/backend/plugins/gitextractor/parser/collector_contract_test.go @@ -0,0 +1,737 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "context" + "sort" + "testing" + + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/models/domainlayer/code" + "github.com/apache/devlake/helpers/unithelper" + mockdal "github.com/apache/devlake/mocks/core/dal" + mockplugin "github.com/apache/devlake/mocks/core/plugin" + "github.com/apache/devlake/plugins/gitextractor/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// collectorCase describes one RepoCollector implementation under test. +// +// The assertions in this file describe the collector contract that a +// replacement implementation must satisfy. The deviations block records where +// an existing implementation is known to violate that contract; they are +// documented instead of being silently tolerated so that a replacement cannot +// inherit them. +type collectorCase struct { + name string + open func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector + + deviations collectorDeviations +} + +type collectorDeviations struct { + // storesOnlyLastComponent reports that only the component of the last file + // of each commit is stored. + storesOnlyLastComponent bool + // countTagsIgnoresContext reports that CountTags ignores a cancelled context. + countTagsIgnoresContext bool + // collectsShallowBoundary reports that the shallow boundary commit is + // collected with a full-tree diff instead of being skipped. + collectsShallowBoundary bool + // failsOnDeletedFiles reports that CollectDiffLine fails when the history + // contains a deleted file. + failsOnDeletedFiles bool + // skipsLineChanges reports that CollectDiffLine does not store + // CommitLineChange rows. + skipsLineChanges bool + // failsOnEmptyRepo reports that the Count* helpers fail on a repository + // without commits. + failsOnEmptyRepo bool + // omitsZeroLineFiles reports that changed files without line changes + // (binary files, empty files) are not stored. + omitsZeroLineFiles bool +} + +func collectorCases() []collectorCase { + return []collectorCase{ + { + name: "libgit2", + open: func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { + t.Helper() + collector, err := NewLibgit2RepoCollector(localDir, repoId, store, unithelper.DummyLogger()) + require.Nil(t, err) + return collector + }, + deviations: collectorDeviations{ + storesOnlyLastComponent: true, + countTagsIgnoresContext: true, + collectsShallowBoundary: true, + }, + }, + { + name: "gogit", + open: func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { + t.Helper() + collector, err := NewGogitRepoCollector(localDir, repoId, store, unithelper.DummyLogger()) + require.Nil(t, err) + return collector + }, + deviations: collectorDeviations{ + failsOnDeletedFiles: true, + skipsLineChanges: true, + failsOnEmptyRepo: true, + omitsZeroLineFiles: true, + }, + }, + } +} + +func forEachCollector(t *testing.T, fn func(t *testing.T, c collectorCase)) { + t.Helper() + for _, c := range collectorCases() { + t.Run(c.name, func(t *testing.T) { + fn(t, c) + }) + } +} + +// recordStore is a models.Store that records everything the collectors write. +type recordStore struct { + incremental bool + closed bool + repoCommits []*code.RepoCommit + commits []*code.Commit + refs []*code.Ref + commitFiles []*code.CommitFile + parents []*code.CommitParent + components []*code.CommitFileComponent + lineChanges []*code.CommitLineChange + snapshots []*code.RepoSnapshot +} + +var _ models.Store = (*recordStore)(nil) + +func (s *recordStore) SetIncrementalMode(v bool) { s.incremental = v } + +func (s *recordStore) RepoCommits(v *code.RepoCommit) errors.Error { + s.repoCommits = append(s.repoCommits, v) + return nil +} + +func (s *recordStore) Commits(v *code.Commit) errors.Error { + cp := *v + s.commits = append(s.commits, &cp) + return nil +} + +func (s *recordStore) Refs(v *code.Ref) errors.Error { + cp := *v + s.refs = append(s.refs, &cp) + return nil +} + +func (s *recordStore) CommitFiles(v *code.CommitFile) errors.Error { + cp := *v + s.commitFiles = append(s.commitFiles, &cp) + return nil +} + +func (s *recordStore) CommitParents(v []*code.CommitParent) errors.Error { + s.parents = append(s.parents, v...) + return nil +} + +func (s *recordStore) CommitFileComponents(v *code.CommitFileComponent) errors.Error { + cp := *v + s.components = append(s.components, &cp) + return nil +} + +func (s *recordStore) CommitLineChange(v *code.CommitLineChange) errors.Error { + cp := *v + s.lineChanges = append(s.lineChanges, &cp) + return nil +} + +func (s *recordStore) RepoSnapshot(v *code.RepoSnapshot) errors.Error { + cp := *v + s.snapshots = append(s.snapshots, &cp) + return nil +} + +func (s *recordStore) Close() errors.Error { + s.closed = true + return nil +} + +func boolPtr(v bool) *bool { return &v } + +func defaultOptions(repoId string) *GitExtractorOptions { + return &GitExtractorOptions{ + GitExtractorApiParams: GitExtractorApiParams{RepoId: repoId}, + SkipCommitStat: boolPtr(false), + SkipCommitFiles: boolPtr(false), + } +} + +// newCollectorContext returns a subtask context backed by a mocked DAL. The +// components are returned from Dal.All, which the collectors use to map files +// to components. +func newCollectorContext(t *testing.T, opts *GitExtractorOptions, components []code.Component) *mockplugin.SubTaskContext { + t.Helper() + dalMock := new(mockdal.Dal) + dalMock.On("All", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + if len(components) == 0 { + return + } + dst, ok := args.Get(0).(*[]code.Component) + if !ok { + return + } + *dst = append(*dst, components...) + }) + dalMock.On("Delete", mock.Anything, mock.Anything).Return(nil) + + ctx := unithelper.DummySubTaskContext(dalMock) + ctx.On("GetContext").Return(context.Background()) + ctx.On("GetData").Return(&GitExtractorTaskData{Options: opts}) + return ctx +} + +func commitsBySha(commits []*code.Commit) map[string]*code.Commit { + out := make(map[string]*code.Commit, len(commits)) + for _, commit := range commits { + out[commit.Sha] = commit + } + return out +} + +func refsByName(refs []*code.Ref) map[string]*code.Ref { + out := make(map[string]*code.Ref, len(refs)) + for _, ref := range refs { + out[ref.Name] = ref + } + return out +} + +func componentsById(components []*code.CommitFileComponent) map[string]string { + out := make(map[string]string, len(components)) + for _, component := range components { + out[component.CommitFileId] = component.ComponentName + } + return out +} + +// filesByCommit returns commit sha -> file path -> [additions, deletions]. +func filesByCommit(files []*code.CommitFile) map[string]map[string][2]int { + out := make(map[string]map[string][2]int) + for _, file := range files { + if out[file.CommitSha] == nil { + out[file.CommitSha] = make(map[string][2]int) + } + out[file.CommitSha][file.FilePath] = [2]int{file.Additions, file.Deletions} + } + return out +} + +func parentShas(parents []*code.CommitParent, commitSha string) []string { + var out []string + for _, parent := range parents { + if parent.CommitSha == commitSha { + out = append(out, parent.ParentCommitSha) + } + } + sort.Strings(out) + return out +} + +func TestCollectorCountsAndRefs(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T, c collectorCase) { + store := &recordStore{} + collector := c.open(t, repo.Dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + commits, err := collector.CountCommits(context.Background()) + require.NoError(t, err) + assert.Equal(t, 5, commits) + + tags, err := collector.CountTags(context.Background()) + require.NoError(t, err) + assert.Equal(t, 2, tags) + // HEAD points at main, which is not counted. + branches, err := collector.CountBranches(context.Background()) + require.NoError(t, err) + assert.Equal(t, 1, branches) + + assert.NoError(t, collector.CollectTags(ctx)) + assert.NoError(t, collector.CollectBranches(ctx)) + + refs := refsByName(store.refs) + require.Len(t, refs, 4) + + // Tag names are stored as the full ref name. + light := refs["refs/tags/v1.0.0"] + require.NotNil(t, light, "lightweight tag v1.0.0") + assert.Equal(t, "test-repo:refs/tags/v1.0.0", light.Id) + assert.Equal(t, repo.C1, light.CommitSha) + assert.Equal(t, TAG, light.RefType) + assert.Equal(t, "test-repo", light.RepoId) + + annotated := refs["refs/tags/v2.0.0"] + require.NotNil(t, annotated, "annotated tag v2.0.0 should resolve to its target commit") + assert.Equal(t, repo.C3, annotated.CommitSha) + assert.Equal(t, TAG, annotated.RefType) + + mainRef := refs["main"] + require.NotNil(t, mainRef) + assert.Equal(t, repo.C5, mainRef.CommitSha) + assert.Equal(t, BRANCH, mainRef.RefType) + assert.True(t, mainRef.IsDefault) + + featureRef := refs["feature"] + require.NotNil(t, featureRef) + assert.Equal(t, repo.C2, featureRef.CommitSha) + assert.Equal(t, BRANCH, featureRef.RefType) + assert.False(t, featureRef.IsDefault) + }) +} + +func TestCollectorCommits(t *testing.T) { + repo := buildCanonicalRepo(t) + repoId := "test-repo" + + forEachCollector(t, func(t *testing.T, c collectorCase) { + store := &recordStore{} + collector := c.open(t, repo.Dir, repoId, store) + ctx := newCollectorContext(t, defaultOptions(repoId), nil) + + require.NoError(t, collector.CollectCommits(ctx)) + + commits := commitsBySha(store.commits) + require.Len(t, commits, 5) + require.Len(t, store.repoCommits, 5) + + for _, tc := range []struct { + sha string + when string + msg string + }{ + {repo.C1, "2026-01-01T00:00:00Z", "c1: initial import"}, + {repo.C2, "2026-01-01T01:00:00Z", "c2: add feature file"}, + {repo.C3, "2026-01-01T02:00:00Z", "c3: reword and extend a.txt"}, + {repo.C4, "2026-01-01T03:00:00Z", "c4: mode and binary change"}, + {repo.C5, "2026-01-01T04:00:00Z", "c5: merge feature"}, + } { + commit := commits[tc.sha] + require.NotNil(t, commit, tc.msg) + assert.Equal(t, tc.msg+"\n", commit.Message) + assert.Equal(t, testAuthorName, commit.AuthorName) + assert.Equal(t, testAuthorEmail, commit.AuthorEmail) + assert.Equal(t, testAuthorEmail, commit.AuthorId) + assert.Equal(t, testAuthorName, commit.CommitterName) + assert.Equal(t, testAuthorEmail, commit.CommitterEmail) + assert.Equal(t, testAuthorEmail, commit.CommitterId) + assert.Equal(t, tc.when, commit.AuthoredDate.UTC().Format("2006-01-02T15:04:05Z")) + assert.Equal(t, tc.when, commit.CommittedDate.UTC().Format("2006-01-02T15:04:05Z")) + } + + assert.Empty(t, parentShas(store.parents, repo.C1)) + assert.Equal(t, []string{repo.C1}, parentShas(store.parents, repo.C2)) + assert.Equal(t, []string{repo.C1}, parentShas(store.parents, repo.C3)) + assert.Equal(t, []string{repo.C3}, parentShas(store.parents, repo.C4)) + assert.ElementsMatch(t, []string{repo.C2, repo.C4}, parentShas(store.parents, repo.C5)) + + files := filesByCommit(store.commitFiles) + expectedFiles := map[string]map[string][2]int{ + repo.C1: { + "a.txt": {3, 0}, + "bin.dat": {0, 0}, + "dir with space/b.txt": {1, 0}, + "empty.txt": {0, 0}, + }, + repo.C2: {"feature.txt": {1, 0}}, + repo.C3: { + "a.txt": {2, 1}, + "empty.txt": {0, 0}, + }, + repo.C4: { + "a.txt": {0, 0}, + "bin.dat": {0, 0}, + }, + // The merge is compared against its first parent, so only the file + // the merge brought in is reported. + repo.C5: {"feature.txt": {1, 0}}, + } + for sha, want := range expectedFiles { + if !c.deviations.omitsZeroLineFiles { + assert.Equal(t, want, files[sha], sha) + continue + } + for path, stat := range want { + if stat != [2]int{0, 0} { + assert.Equal(t, stat, files[sha][path], "%s %s", sha, path) + } + } + for path, stat := range files[sha] { + if stat != [2]int{0, 0} { + assert.Contains(t, want, path, "%s %s", sha, path) + } + } + } + + assert.Equal(t, 4, commits[repo.C1].Additions) + assert.Equal(t, 0, commits[repo.C1].Deletions) + assert.Equal(t, 1, commits[repo.C2].Additions) + assert.Equal(t, 2, commits[repo.C3].Additions) + assert.Equal(t, 1, commits[repo.C3].Deletions) + assert.Equal(t, 0, commits[repo.C4].Additions) + assert.Equal(t, 0, commits[repo.C4].Deletions) + assert.Equal(t, 1, commits[repo.C5].Additions) + assert.Equal(t, 0, commits[repo.C5].Deletions) + + for _, file := range store.commitFiles { + assert.Equal(t, genCommitFileId(file.CommitSha, file.FilePath), file.Id) + } + if !c.deviations.storesOnlyLastComponent { + components := componentsById(store.components) + for _, file := range store.commitFiles { + assert.Equal(t, "Default", components[file.Id], file.FilePath) + } + } + }) +} + +func TestCollectorUsesConfiguredComponents(t *testing.T) { + repo := buildLinearRepo(t) + + forEachCollector(t, func(t *testing.T, c collectorCase) { + store := &recordStore{} + collector := c.open(t, repo.Dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), []code.Component{ + {RepoId: "test-repo", Name: "docs", PathRegex: `^c\.txt$`}, + }) + + require.NoError(t, collector.CollectCommits(ctx)) + + components := componentsById(store.components) + files := filesByCommit(store.commitFiles)[repo.C4] + require.Contains(t, files, "c.txt") + assert.Equal(t, "docs", components[genCommitFileId(repo.C4, "c.txt")]) + if !c.deviations.storesOnlyLastComponent { + assert.Equal(t, "Default", components[genCommitFileId(repo.C4, "b.txt")]) + } + }) +} + +func TestCollectorSkipCommitStat(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T, c collectorCase) { + store := &recordStore{} + collector := c.open(t, repo.Dir, "test-repo", store) + opts := defaultOptions("test-repo") + opts.SkipCommitStat = boolPtr(true) + opts.SkipCommitFiles = boolPtr(true) + ctx := newCollectorContext(t, opts, nil) + + require.NoError(t, collector.CollectCommits(ctx)) + + require.Len(t, store.commits, 5) + for _, commit := range store.commits { + assert.Zero(t, commit.Additions) + assert.Zero(t, commit.Deletions) + } + assert.Empty(t, store.commitFiles) + assert.Empty(t, store.components) + }) +} + +func TestCollectorSkipCommitFiles(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T, c collectorCase) { + store := &recordStore{} + collector := c.open(t, repo.Dir, "test-repo", store) + opts := defaultOptions("test-repo") + opts.SkipCommitFiles = boolPtr(true) + ctx := newCollectorContext(t, opts, nil) + + require.NoError(t, collector.CollectCommits(ctx)) + + assert.Empty(t, store.commitFiles) + assert.Empty(t, store.components) + commits := commitsBySha(store.commits) + assert.Equal(t, 4, commits[repo.C1].Additions) + assert.Equal(t, 1, commits[repo.C2].Additions) + }) +} + +func TestCollectorExcludeFileExtensions(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T, c collectorCase) { + store := &recordStore{} + collector := c.open(t, repo.Dir, "test-repo", store) + opts := defaultOptions("test-repo") + opts.ExcludeFileExtensions = []string{".DAT"} + ctx := newCollectorContext(t, opts, nil) + + require.NoError(t, collector.CollectCommits(ctx)) + + commits := commitsBySha(store.commits) + assert.Equal(t, 4, commits[repo.C1].Additions) + assert.Equal(t, 0, commits[repo.C4].Additions) + assert.Equal(t, 0, commits[repo.C4].Deletions) + + files := filesByCommit(store.commitFiles) + assert.NotContains(t, files[repo.C1], "bin.dat") + assert.NotContains(t, files[repo.C4], "bin.dat") + assert.Contains(t, files[repo.C1], "a.txt") + }) +} + +func TestCollectorSkipsShallowBoundaryCommit(t *testing.T) { + f := newGitFixture(t) + f.writeFile("a.txt", "l1\n") + c1 := f.commit(testTimeC1, "c1") + f.writeFile("a.txt", "l1\nl2\n") + c2 := f.commit(testTimeC2, "c2") + require.NotEqual(t, c1, c2) + + dir := f.shallowBareClone() + + forEachCollector(t, func(t *testing.T, c collectorCase) { + if c.deviations.collectsShallowBoundary { + t.Skip("known deviation: the shallow boundary is collected with a full-tree diff") + } + store := &recordStore{} + collector := c.open(t, dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + // The tip is a shallow boundary: its parent is not in the object + // database, so the collector must not compute stats for it (a full-tree + // diff would store an absurdly large addition count). + require.NoError(t, collector.CollectCommits(ctx)) + assert.Empty(t, store.commits) + assert.Empty(t, store.commitFiles) + }) +} + +func TestCollectorEmptyRepo(t *testing.T) { + dir := newGitFixture(t).bareClone() + + forEachCollector(t, func(t *testing.T, c collectorCase) { + if c.deviations.failsOnEmptyRepo { + t.Skip("known deviation: the Count* helpers need at least one commit") + } + store := &recordStore{} + collector := c.open(t, dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + commits, err := collector.CountCommits(context.Background()) + require.NoError(t, err) + assert.Zero(t, commits) + tags, err := collector.CountTags(context.Background()) + require.NoError(t, err) + assert.Zero(t, tags) + branches, err := collector.CountBranches(context.Background()) + require.NoError(t, err) + assert.Zero(t, branches) + + assert.NoError(t, collector.CollectTags(ctx)) + assert.NoError(t, collector.CollectBranches(ctx)) + assert.NoError(t, collector.CollectCommits(ctx)) + assert.Empty(t, store.refs) + assert.Empty(t, store.commits) + }) +} + +func TestCollectorHonoursCancelledContext(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T, c collectorCase) { + store := &recordStore{} + collector := c.open(t, repo.Dir, "test-repo", store) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := collector.CountCommits(ctx) + assert.ErrorIs(t, err, context.Canceled) + if !c.deviations.countTagsIgnoresContext { + _, err = collector.CountTags(ctx) + assert.ErrorIs(t, err, context.Canceled) + } + _, err = collector.CountBranches(ctx) + assert.ErrorIs(t, err, context.Canceled) + }) +} + +func TestCollectorRejectsNonRepo(t *testing.T) { + dir := t.TempDir() + for _, c := range collectorCases() { + t.Run(c.name, func(t *testing.T) { + var err error + switch c.name { + case "libgit2": + _, err = NewLibgit2RepoCollector(dir, "test-repo", nil, unithelper.DummyLogger()) + case "gogit": + _, err = NewGogitRepoCollector(dir, "test-repo", nil, unithelper.DummyLogger()) + } + assert.Error(t, err, "%s should reject a non-repository", c.name) + }) + } +} + +func TestCollectorCountsWhenHeadIsNotDefault(t *testing.T) { + f := newGitFixture(t) + f.writeFile("a.txt", "l1\n") + c1 := f.commit(testTimeC1, "c1") + f.checkout("feature", true) + f.writeFile("a.txt", "l1\nl2\n") + c2 := f.commit(testTimeC2, "c2") + require.NotEqual(t, c1, c2) + dir := f.bareClone() + + forEachCollector(t, func(t *testing.T, c collectorCase) { + store := &recordStore{} + collector := c.open(t, dir, "test-repo", store) + + // HEAD points at feature, which is not counted but is the default ref. + branches, err := collector.CountBranches(context.Background()) + require.NoError(t, err) + assert.Equal(t, 1, branches) + + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + require.NoError(t, collector.CollectBranches(ctx)) + + refs := refsByName(store.refs) + require.Len(t, refs, 2) + assert.True(t, refs["feature"].IsDefault) + assert.False(t, refs["main"].IsDefault) + }) +} + +// TestCollectorDiffLineSnapshot checks the repo snapshot against git blame, +// which is an independent oracle for "which commit introduced each line". +func TestCollectorDiffLineSnapshot(t *testing.T) { + repo := buildLinearRepo(t) + expected := blameAtHEAD(t, repo.Dir) + + forEachCollector(t, func(t *testing.T, c collectorCase) { + if c.deviations.failsOnDeletedFiles { + t.Skip("known deviation: CollectDiffLine fails on a deleted file") + } + store := &recordStore{} + collector := c.open(t, repo.Dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + require.NoError(t, collector.CollectDiffLine(ctx)) + + got := make(map[string]map[int]string) + for _, snapshot := range store.snapshots { + if got[snapshot.FilePath] == nil { + got[snapshot.FilePath] = make(map[int]string) + } + got[snapshot.FilePath][snapshot.LineNo] = snapshot.CommitSha + } + + // b.txt is deleted in c4, so it has no lines in HEAD and no snapshot rows. + assert.NotContains(t, got, "b.txt") + for file, lines := range expected { + assert.Equal(t, lines, got[file], file) + } + + if c.deviations.skipsLineChanges { + assert.Empty(t, store.lineChanges) + } + }) +} + +func TestCollectorCollectAllAndClose(t *testing.T) { + repo := buildCanonicalRepo(t) + + forEachCollector(t, func(t *testing.T, c collectorCase) { + if c.deviations.failsOnDeletedFiles { + t.Skip("known deviation: CollectDiffLine fails on a deleted file") + } + store := &recordStore{} + collector := c.open(t, repo.Dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + cleanedUp := false + require.NoError(t, collector.SetCleanUp(func() { cleanedUp = true })) + + require.NoError(t, collector.CollectAll(ctx)) + assert.Len(t, store.commits, 5) + assert.Len(t, store.repoCommits, 5) + assert.Len(t, store.refs, 4) + assert.NotEmpty(t, store.commitFiles) + assert.NotEmpty(t, store.snapshots) + if c.deviations.skipsLineChanges { + assert.Empty(t, store.lineChanges) + } else { + assert.NotEmpty(t, store.lineChanges) + } + + require.NoError(t, collector.Close(context.Background())) + assert.True(t, store.closed) + assert.True(t, cleanedUp) + }) +} + +func TestLibgit2CollectorCommitLineChanges(t *testing.T) { + repo := buildLinearRepo(t) + + store := &recordStore{} + collector, err := NewLibgit2RepoCollector(repo.Dir, "test-repo", store, unithelper.DummyLogger()) + require.Nil(t, err) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + require.NoError(t, collector.CollectDiffLine(ctx)) + require.NotEmpty(t, store.lineChanges) + + // c3 appends line 4 to a.txt, so a.txt gets one addition at new line 4. + var additions []*code.CommitLineChange + for _, change := range store.lineChanges { + if change.CommitSha == repo.C3 && change.NewFilePath == "a.txt" && change.ChangedType == "Addition" { + additions = append(additions, change) + } + } + require.Len(t, additions, 1) + assert.Equal(t, 4, additions[0].LineNoNew) + assert.Equal(t, -1, additions[0].LineNoOld) + assert.Equal(t, 1, additions[0].HunkNum) + assert.Equal(t, "", additions[0].PrevCommit) + assert.Equal(t, repo.C3+":a.txt:-1:4", additions[0].Id) + + // c2 rewrites line 2 of a.txt; the deletion must be attributed to c1. + var deletions []*code.CommitLineChange + for _, change := range store.lineChanges { + if change.CommitSha == repo.C2 && change.NewFilePath == "a.txt" && change.ChangedType == "Deletion" { + deletions = append(deletions, change) + } + } + require.Len(t, deletions, 1) + assert.Equal(t, 2, deletions[0].LineNoOld) + assert.Equal(t, repo.C1, deletions[0].PrevCommit) +} diff --git a/backend/plugins/gitextractor/parser/repo_gogit_test.go b/backend/plugins/gitextractor/parser/repo_gogit_test.go deleted file mode 100644 index a4dfa321b..000000000 --- a/backend/plugins/gitextractor/parser/repo_gogit_test.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package parser - -import ( - "context" - "os" - "path/filepath" - "testing" - "time" - - gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// buildTestRepo creates a repository on disk with a known shape: 3 commits on -// the default branch, one lightweight tag and a second branch which is checked -// out at the end, so HEAD does not point at the default branch. -func buildTestRepo(t *testing.T) string { - t.Helper() - dir := t.TempDir() - - repo, err := gogit.PlainInit(dir, false) - require.NoError(t, err) - wt, err := repo.Worktree() - require.NoError(t, err) - - sig := func(n int) *object.Signature { - return &object.Signature{ - Name: "DevLake Test", - Email: "[email protected]", - When: time.Date(2026, 1, 1, 0, 0, n, 0, time.UTC), - } - } - - var last plumbing.Hash - for i := 1; i <= 3; i++ { - name := filepath.Join(dir, "file.txt") - require.NoError(t, os.WriteFile(name, []byte{byte('a' + i)}, 0600)) - _, err = wt.Add("file.txt") - require.NoError(t, err) - last, err = wt.Commit("commit", &gogit.CommitOptions{Author: sig(i)}) - require.NoError(t, err) - } - - _, err = repo.CreateTag("v1.0.0", last, nil) - require.NoError(t, err) - - require.NoError(t, wt.Checkout(&gogit.CheckoutOptions{ - Branch: plumbing.NewBranchReferenceName("feature"), - Create: true, - })) - - return dir -} - -func TestGogitRepoCollectorCounts(t *testing.T) { - dir := buildTestRepo(t) - - // store and logger are only used by the Collect* subtasks, not by the - // counting helpers exercised here. - collector, err := NewGogitRepoCollector(dir, "test-repo", nil, nil) - require.Nil(t, err) - - ctx := context.Background() - - commits, cErr := collector.CountCommits(ctx) - assert.NoError(t, cErr) - assert.Equal(t, 3, commits) - - tags, tErr := collector.CountTags(ctx) - assert.NoError(t, tErr) - assert.Equal(t, 1, tags) - - // HEAD is on "feature", so only the default branch is counted. - branches, bErr := collector.CountBranches(ctx) - assert.NoError(t, bErr) - assert.Equal(t, 1, branches) -} - -func TestGogitRepoCollectorCountsHonourCancelledContext(t *testing.T) { - dir := buildTestRepo(t) - - collector, err := NewGogitRepoCollector(dir, "test-repo", nil, nil) - require.Nil(t, err) - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - _, cErr := collector.CountCommits(ctx) - assert.ErrorIs(t, cErr, context.Canceled) - - _, tErr := collector.CountTags(ctx) - assert.ErrorIs(t, tErr, context.Canceled) - - _, bErr := collector.CountBranches(ctx) - assert.ErrorIs(t, bErr, context.Canceled) -} - -func TestNewGogitRepoCollectorRejectsNonRepo(t *testing.T) { - _, err := NewGogitRepoCollector(t.TempDir(), "test-repo", nil, nil) - assert.NotNil(t, err) -} - diff --git a/backend/plugins/gitextractor/parser/testrepo_test.go b/backend/plugins/gitextractor/parser/testrepo_test.go new file mode 100644 index 000000000..400acd637 --- /dev/null +++ b/backend/plugins/gitextractor/parser/testrepo_test.go @@ -0,0 +1,316 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// This file builds deterministic git repositories with the git command line. +// The collector tests use them as fixtures so that the fixtures do not depend +// on any of the git libraries under test. + +const ( + testAuthorName = "DevLake Test" + testAuthorEmail = "[email protected]" +) + +var ( + testTimeBase = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + testTimeC1 = testTimeBase + testTimeC2 = testTimeBase.Add(1 * time.Hour) + testTimeC3 = testTimeBase.Add(2 * time.Hour) + testTimeC4 = testTimeBase.Add(3 * time.Hour) + testTimeC5 = testTimeBase.Add(4 * time.Hour) +) + +func requireGit(t *testing.T) { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git is not installed") + } +} + +// gitFixture is a scriptable git repository. Commands run with an isolated +// git configuration and a fixed identity so the fixtures are reproducible. +type gitFixture struct { + t *testing.T + dir string +} + +func newGitFixture(t *testing.T) *gitFixture { + t.Helper() + requireGit(t) + f := newGitFixtureAt(t, t.TempDir()) + f.git("init", "-q", "-b", "main") + return f +} + +func newGitFixtureAt(t *testing.T, dir string) *gitFixture { + t.Helper() + requireGit(t) + return &gitFixture{t: t, dir: dir} +} + +func (f *gitFixture) env(when time.Time) []string { + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+testAuthorName, + "GIT_AUTHOR_EMAIL="+testAuthorEmail, + "GIT_COMMITTER_NAME="+testAuthorName, + "GIT_COMMITTER_EMAIL="+testAuthorEmail, + "GIT_CONFIG_NOSYSTEM=1", + "GIT_CONFIG_GLOBAL="+os.DevNull, + "GIT_TERMINAL_PROMPT=0", + "TZ=UTC", + // keep the fixtures independent of the host user configuration + "HOME="+f.dir, + ) + if !when.IsZero() { + date := when.UTC().Format(time.RFC3339) + env = append(env, "GIT_AUTHOR_DATE="+date, "GIT_COMMITTER_DATE="+date) + } + return env +} + +func (f *gitFixture) git(args ...string) string { + f.t.Helper() + return f.gitAt(time.Time{}, args...) +} + +func (f *gitFixture) gitAt(when time.Time, args ...string) string { + f.t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = f.dir + cmd.Env = f.env(when) + output, err := cmd.CombinedOutput() + require.NoError(f.t, err, "git %v in %s: %s", args, f.dir, output) + return strings.TrimSpace(string(output)) +} + +func (f *gitFixture) head() string { + f.t.Helper() + return f.git("rev-parse", "HEAD") +} + +func (f *gitFixture) writeFile(rel, content string) { + f.t.Helper() + full := filepath.Join(f.dir, rel) + require.NoError(f.t, os.MkdirAll(filepath.Dir(full), 0o755)) + require.NoError(f.t, os.WriteFile(full, []byte(content), 0o644)) +} + +func (f *gitFixture) chmod(rel string, mode os.FileMode) { + f.t.Helper() + require.NoError(f.t, os.Chmod(filepath.Join(f.dir, rel), mode)) +} + +func (f *gitFixture) removeFile(rel string) { + f.t.Helper() + require.NoError(f.t, os.Remove(filepath.Join(f.dir, rel))) +} + +// commit stages everything and commits with the fixture identity at when. +func (f *gitFixture) commit(when time.Time, message string) string { + f.t.Helper() + f.git("add", "-A") + f.gitAt(when, "commit", "-q", "-m", message) + return f.head() +} + +func (f *gitFixture) checkout(branch string, create bool) { + f.t.Helper() + if create { + f.git("checkout", "-q", "-b", branch) + return + } + f.git("checkout", "-q", branch) +} + +func (f *gitFixture) merge(when time.Time, message, branch string) string { + f.t.Helper() + f.gitAt(when, "merge", "-q", "--no-ff", "-m", message, branch) + return f.head() +} + +func (f *gitFixture) tag(name, target string, annotated bool) { + f.t.Helper() + if annotated { + f.git("tag", "-a", name, target, "-m", "release "+name) + return + } + f.git("tag", name, target) +} + +// bareClone clones the fixture into a new bare repository, which is the shape +// the gitextractor produces before collecting. +func (f *gitFixture) bareClone() string { + f.t.Helper() + dir := filepath.Join(f.t.TempDir(), "repo.git") + f.git("clone", "-q", "--bare", f.dir, dir) + return dir +} + +// shallowBareClone creates a depth-1 bare clone, so the tip commit is a +// shallow boundary whose parent is missing from the object database. +func (f *gitFixture) shallowBareClone() string { + f.t.Helper() + dir := filepath.Join(f.t.TempDir(), "shallow.git") + f.git("clone", "-q", "--bare", "--depth=1", "file://"+f.dir, dir) + return dir +} + +// canonicalRepo is the repository used by the collector contract tests. +// +// main: c1 ---- c3 ---- c4 ---- c5 (merge) +// \ / +// feature: c2 ---------------- +// +// c1 adds a.txt (3 lines), a file in a directory with a space, a binary file +// and an empty file. c2 adds a file on the feature branch. c3 rewrites a line +// in a.txt, appends a line and deletes the empty file. c4 only changes the file +// mode of a.txt and rewrites the binary file. c5 merges feature. +// c1 carries the lightweight tag v1.0.0, c3 the annotated tag v2.0.0. +type canonicalRepo struct { + Dir string + C1 string + C2 string + C3 string + C4 string + C5 string +} + +func buildCanonicalRepo(t *testing.T) canonicalRepo { + t.Helper() + f := newGitFixture(t) + + f.writeFile("a.txt", "l1\nl2\nl3\n") + f.writeFile("dir with space/b.txt", "b1\n") + f.writeFile("bin.dat", "before\x00binary\n") + f.writeFile("empty.txt", "") + c1 := f.commit(testTimeC1, "c1: initial import") + + f.checkout("feature", true) + f.writeFile("feature.txt", "f1\n") + c2 := f.commit(testTimeC2, "c2: add feature file") + + f.checkout("main", false) + f.writeFile("a.txt", "l1\nL2\nl3\nl4\n") + f.removeFile("empty.txt") + c3 := f.commit(testTimeC3, "c3: reword and extend a.txt") + + f.chmod("a.txt", 0o755) + f.writeFile("bin.dat", "after\x00binary\n") + c4 := f.commit(testTimeC4, "c4: mode and binary change") + + c5 := f.merge(testTimeC5, "c5: merge feature", "feature") + + f.tag("v1.0.0", c1, false) + f.tag("v2.0.0", c3, true) + + return canonicalRepo{ + Dir: f.bareClone(), + C1: c1, + C2: c2, + C3: c3, + C4: c4, + C5: c5, + } +} + +// buildLinearRepo creates a repository without merges for the diff line tests. +// +// c1 adds a.txt and b.txt, c2 rewrites a line in a.txt, c3 appends a line to +// a.txt and rewrites a line in b.txt, c4 deletes b.txt and adds c.txt. +type linearRepo struct { + Dir string + C1 string + C2 string + C3 string + C4 string +} + +func buildLinearRepo(t *testing.T) linearRepo { + t.Helper() + f := newGitFixture(t) + + f.writeFile("a.txt", "l1\nl2\nl3\n") + f.writeFile("b.txt", "b1\nb2\n") + c1 := f.commit(testTimeC1, "c1") + + f.writeFile("a.txt", "l1\nL2\nl3\n") + c2 := f.commit(testTimeC2, "c2") + + f.writeFile("a.txt", "l1\nL2\nl3\nl4\n") + f.writeFile("b.txt", "b1\nB2\n") + c3 := f.commit(testTimeC3, "c3") + + f.removeFile("b.txt") + f.writeFile("c.txt", "c1\nc2\n") + c4 := f.commit(testTimeC4, "c4") + + return linearRepo{ + Dir: f.bareClone(), + C1: c1, + C2: c2, + C3: c3, + C4: c4, + } +} + +// blameAtHEAD returns file -> line number -> commit sha for every file in HEAD, +// using the git command line as an independent oracle. +func blameAtHEAD(t *testing.T, dir string) map[string]map[int]string { + t.Helper() + repo := newGitFixtureAt(t, dir) + blames := make(map[string]map[int]string) + files := strings.Split(strings.TrimRight(repo.git("ls-tree", "-r", "--name-only", "-z", "HEAD"), "\x00"), "\x00") + for _, file := range files { + if file == "" { + continue + } + out := repo.git("blame", "-l", "--line-porcelain", "HEAD", "--", file) + lines := make(map[int]string) + for _, line := range strings.Split(out, "\n") { + // The header of the first line of each group carries the group + // size, later headers do not. Content lines are tab-prefixed. + if strings.HasPrefix(line, "\t") { + continue + } + fields := strings.Fields(line) + if len(fields) < 3 || len(fields[0]) != 40 { + continue + } + finalLine, err := strconv.Atoi(fields[2]) + if err != nil { + continue + } + lines[finalLine] = fields[0] + } + blames[file] = lines + } + return blames +}
