This is an automated email from the ASF dual-hosted git repository. klesh pushed a commit to branch feat/gitextractor-git-cli in repository https://gitbox.apache.org/repos/asf/devlake.git
commit 517a0a175ae94d80012484f01b342732797da276 Author: Klesh Wong <[email protected]> AuthorDate: Sun Nov 1 13:13:59 2026 +0800 feat(gitextractor): add git CLI collector Implement RepoCollector on top of git plumbing and porcelain commands and select it with GIT_EXTRACTOR_COLLECTOR=gitcli (or --useGitCli) for A/B testing. libgit2 and go-git stay the default for now. The collector streams rev-list, cat-file and numstat output in bounded batches, skips shallow boundary commits, honours context cancellation, and rebuilds the line snapshot from the first-parent patch stream. The contract suite now runs against all three collectors; the git CLI collector passes without any of the documented deviations. On apache/devlake (8383 reachable commits) CollectCommits takes 2.9s versus 21.3s with libgit2, and CollectDiffLine produces 559k snapshot rows and 2.7M line changes in 3.4s. --- backend/plugins/gitextractor/gitextractor.go | 2 + backend/plugins/gitextractor/impl/impl.go | 3 + .../plugins/gitextractor/parser/clone_gitcli.go | 2 + .../gitextractor/parser/collector_contract_test.go | 11 + backend/plugins/gitextractor/parser/commitparse.go | 172 +++++ .../gitextractor/parser/commitparse_test.go | 115 ++++ backend/plugins/gitextractor/parser/gitrunner.go | 155 +++++ .../plugins/gitextractor/parser/numstatparse.go | 130 ++++ backend/plugins/gitextractor/parser/parse_test.go | 243 +++++++ backend/plugins/gitextractor/parser/patchparse.go | 332 +++++++++ backend/plugins/gitextractor/parser/repo_gitcli.go | 753 +++++++++++++++++++++ backend/plugins/gitextractor/parser/taskdata.go | 4 + backend/plugins/gitextractor/tasks/repo_cloner.go | 4 +- 13 files changed, 1925 insertions(+), 1 deletion(-) diff --git a/backend/plugins/gitextractor/gitextractor.go b/backend/plugins/gitextractor/gitextractor.go index 67e0b7f0f..1261c3834 100644 --- a/backend/plugins/gitextractor/gitextractor.go +++ b/backend/plugins/gitextractor/gitextractor.go @@ -36,6 +36,7 @@ func main() { // pkPass := cmd.Flags().StringP("privateKeyPassPhrase", "P", "", "passphrase for private key") proxy := cmd.Flags().StringP("proxy", "x", "", "proxy") useGoGit := cmd.Flags().BoolP("useGoGit", "g", false, "use go-git instead of libgit2") + useGitCli := cmd.Flags().BoolP("useGitCli", "", false, "use the git command line collector") skipCommitStat := cmd.Flags().BoolP("skipCommitStat", "S", false, "") skipCommitFiles := cmd.Flags().BoolP("skipCommitFiles", "F", true, "") noShallowClone := cmd.Flags().BoolP("noShallowClone", "A", false, "") @@ -53,6 +54,7 @@ func main() { // "passphrase" "proxy": *proxy, "useGoGit": *useGoGit, + "useGitCli": *useGitCli, "skipCommitStat": skipCommitStat, "skipCommitFiles": skipCommitFiles, "noShallowClone": noShallowClone, diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go index d0f763489..746afee85 100644 --- a/backend/plugins/gitextractor/impl/impl.go +++ b/backend/plugins/gitextractor/impl/impl.go @@ -20,6 +20,7 @@ package impl import ( "fmt" "net/url" + "strings" "github.com/apache/devlake/core/dal" "github.com/apache/devlake/core/errors" @@ -118,7 +119,9 @@ func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[st loadBool(&op.UseGoGit, "UseGoGit", false) loadBool(&op.SkipCommitStat, "SKIP_COMMIT_STAT", false) loadBool(&op.SkipCommitFiles, "SKIP_COMMIT_FILES", true) + op.UseGitCli = op.UseGitCli || strings.EqualFold(cfg.GetString("GIT_EXTRACTOR_COLLECTOR"), "gitcli") log.Info("UseGoGit: %v", *op.UseGoGit) + log.Info("UseGitCli: %v", op.UseGitCli) log.Info("SkipCommitStat: %v", *op.SkipCommitStat) log.Info("SkipCommitFiles: %v", *op.SkipCommitFiles) diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go index 82fc9d57a..3c88899f7 100644 --- a/backend/plugins/gitextractor/parser/clone_gitcli.go +++ b/backend/plugins/gitextractor/parser/clone_gitcli.go @@ -47,6 +47,7 @@ const maxExtraDeepenRounds = 10 // the subtask should run in Full Sync mode whenever the configuration is changed type CloneRepoConfig struct { UseGoGit *bool + UseGitCli bool SkipCommitStat *bool SkipCommitFiles *bool NoShallowClone bool @@ -73,6 +74,7 @@ func NewGitcliCloner(ctx plugin.SubTaskContext, localDir string) (*GitcliCloner, Params: taskData.Options.GitExtractorApiParams, SubtaskConfig: CloneRepoConfig{ UseGoGit: taskData.Options.UseGoGit, + UseGitCli: taskData.Options.UseGitCli, SkipCommitStat: taskData.Options.SkipCommitStat, SkipCommitFiles: taskData.Options.SkipCommitFiles, NoShallowClone: taskData.Options.NoShallowClone, diff --git a/backend/plugins/gitextractor/parser/collector_contract_test.go b/backend/plugins/gitextractor/parser/collector_contract_test.go index 8ec51598a..52b97ece9 100644 --- a/backend/plugins/gitextractor/parser/collector_contract_test.go +++ b/backend/plugins/gitextractor/parser/collector_contract_test.go @@ -72,6 +72,15 @@ type collectorDeviations struct { func collectorCases() []collectorCase { return []collectorCase{ + { + name: "gitcli", + open: func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { + t.Helper() + collector, err := NewGitcliRepoCollector(localDir, repoId, store, unithelper.DummyLogger()) + require.Nil(t, err) + return collector + }, + }, { name: "libgit2", open: func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { @@ -592,6 +601,8 @@ func TestCollectorRejectsNonRepo(t *testing.T) { t.Run(c.name, func(t *testing.T) { var err error switch c.name { + case "gitcli": + _, err = NewGitcliRepoCollector(dir, "test-repo", nil, unithelper.DummyLogger()) case "libgit2": _, err = NewLibgit2RepoCollector(dir, "test-repo", nil, unithelper.DummyLogger()) case "gogit": diff --git a/backend/plugins/gitextractor/parser/commitparse.go b/backend/plugins/gitextractor/parser/commitparse.go new file mode 100644 index 000000000..4f52f89f8 --- /dev/null +++ b/backend/plugins/gitextractor/parser/commitparse.go @@ -0,0 +1,172 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strconv" + "strings" + "time" +) + +// parsedCommit is the metadata of one commit object, as stored by git. +type parsedCommit struct { + Sha string + Parents []string + Author parsedSignature + Committer parsedSignature + Message string +} + +type parsedSignature struct { + Name string + Email string + When time.Time +} + +// parseCommitObject parses the raw content of a commit object. +func parseCommitObject(sha string, content []byte) (*parsedCommit, error) { + commit := &parsedCommit{Sha: sha} + + header := content + if idx := bytes.Index(content, []byte("\n\n")); idx >= 0 { + header = content[:idx] + commit.Message = string(content[idx+2:]) + } + + for _, line := range strings.Split(string(header), "\n") { + if line == "" { + continue + } + if line[0] == ' ' { + // continuation of a multi-line header such as gpgsig, not used here + continue + } + key, value, ok := strings.Cut(line, " ") + if !ok { + continue + } + switch key { + case "parent": + commit.Parents = append(commit.Parents, value) + case "author": + signature, err := parseSignature(value) + if err != nil { + return nil, fmt.Errorf("commit %s: %w", sha, err) + } + commit.Author = signature + case "committer": + signature, err := parseSignature(value) + if err != nil { + return nil, fmt.Errorf("commit %s: %w", sha, err) + } + commit.Committer = signature + } + } + return commit, nil +} + +// parseSignature parses "Name <email> <unix timestamp> <timezone>". +func parseSignature(value string) (parsedSignature, error) { + var signature parsedSignature + + tzSpace := strings.LastIndexByte(value, ' ') + if tzSpace < 0 { + return signature, fmt.Errorf("invalid signature %q", value) + } + timezone := value[tzSpace+1:] + rest := value[:tzSpace] + + tsSpace := strings.LastIndexByte(rest, ' ') + if tsSpace < 0 { + return signature, fmt.Errorf("invalid signature %q", value) + } + seconds, err := strconv.ParseInt(rest[tsSpace+1:], 10, 64) + if err != nil { + return signature, fmt.Errorf("invalid timestamp in signature %q", value) + } + identity := rest[:tsSpace] + + if lt := strings.LastIndexByte(identity, '<'); lt >= 0 { + if gt := strings.LastIndexByte(identity[lt:], '>'); gt >= 0 { + signature.Email = identity[lt+1 : lt+gt] + signature.Name = strings.TrimSpace(identity[:lt]) + } else { + signature.Name = strings.TrimSpace(identity) + } + } else { + signature.Name = strings.TrimSpace(identity) + } + + signature.When = time.Unix(seconds, 0).In(parseTimezone(timezone)) + return signature, nil +} + +// parseTimezone converts a git timezone such as "+0800" into a fixed zone so +// the original offset of the commit date is preserved. +func parseTimezone(timezone string) *time.Location { + if len(timezone) != 5 || (timezone[0] != '+' && timezone[0] != '-') { + return time.UTC + } + hours, errH := strconv.Atoi(timezone[1:3]) + minutes, errM := strconv.Atoi(timezone[3:5]) + if errH != nil || errM != nil { + return time.UTC + } + offset := hours*3600 + minutes*60 + if timezone[0] == '-' { + offset = -offset + } + return time.FixedZone(timezone, offset) +} + +// readCatFileObject reads one object from `git cat-file --batch` output. +// It returns io.EOF when the stream is exhausted. +func readCatFileObject(reader *bufio.Reader) (string, []byte, error) { + header, err := reader.ReadString('\n') + if err == io.EOF && header == "" { + return "", nil, io.EOF + } + if err != nil && err != io.EOF { + return "", nil, err + } + + fields := strings.Fields(header) + if len(fields) != 3 { + return "", nil, fmt.Errorf("unexpected cat-file header %q", strings.TrimSpace(header)) + } + if fields[1] != "commit" { + return "", nil, fmt.Errorf("object %s is a %s, not a commit", fields[0], fields[1]) + } + size, err := strconv.Atoi(fields[2]) + if err != nil { + return "", nil, fmt.Errorf("unexpected cat-file size in %q", strings.TrimSpace(header)) + } + + content := make([]byte, size) + if _, err := io.ReadFull(reader, content); err != nil { + return "", nil, err + } + if _, err := reader.Discard(1); err != nil { // trailing newline + return "", nil, err + } + return fields[0], content, nil +} diff --git a/backend/plugins/gitextractor/parser/commitparse_test.go b/backend/plugins/gitextractor/parser/commitparse_test.go new file mode 100644 index 000000000..473fc070c --- /dev/null +++ b/backend/plugins/gitextractor/parser/commitparse_test.go @@ -0,0 +1,115 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "io" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const rawCommitObject = `tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904 +parent 1111111111111111111111111111111111111111 +parent 2222222222222222222222222222222222222222 +author Jane Doe <[email protected]> 1704067200 +0800 +committer DevLake Test <[email protected]> 1704070800 -0530 +gpgsig -----BEGIN PGP SIGNATURE----- + + fakesignature + -----END PGP SIGNATURE----- + +subject line + +body line 1 +body line 2 +` + +func TestParseCommitObject(t *testing.T) { + commit, err := parseCommitObject("abc", []byte(rawCommitObject)) + require.NoError(t, err) + + assert.Equal(t, "abc", commit.Sha) + assert.Equal(t, []string{ + "1111111111111111111111111111111111111111", + "2222222222222222222222222222222222222222", + }, commit.Parents) + + assert.Equal(t, "Jane Doe", commit.Author.Name) + assert.Equal(t, "[email protected]", commit.Author.Email) + assert.Equal(t, int64(1704067200), commit.Author.When.Unix()) + _, offset := commit.Author.When.Zone() + assert.Equal(t, 8*3600, offset) + + assert.Equal(t, "DevLake Test", commit.Committer.Name) + assert.Equal(t, "[email protected]", commit.Committer.Email) + _, offset = commit.Committer.When.Zone() + assert.Equal(t, -(5*3600 + 30*60), offset) + + assert.Equal(t, "subject line\n\nbody line 1\nbody line 2\n", commit.Message) +} + +func TestParseCommitObjectWithoutMessage(t *testing.T) { + content := "tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n" + + "author A <a@x> 1704067200 +0000\n" + + "committer A <a@x> 1704067200 +0000\n" + + commit, err := parseCommitObject("abc", []byte(content)) + require.NoError(t, err) + assert.Empty(t, commit.Parents) + assert.Empty(t, commit.Message) + assert.Equal(t, time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), commit.Author.When.UTC()) +} + +func TestParseSignature(t *testing.T) { + signature, err := parseSignature("A U Thor <[email protected]> 1704067200 +0000") + require.NoError(t, err) + assert.Equal(t, "A U Thor", signature.Name) + assert.Equal(t, "[email protected]", signature.Email) + + // a name containing '<' still resolves to the last email-like part + signature, err = parseSignature("Weird <Name> <[email protected]> 1704067200 +0000") + require.NoError(t, err) + assert.Equal(t, "Weird <Name>", signature.Name) + assert.Equal(t, "[email protected]", signature.Email) + + _, err = parseSignature("no timestamp") + assert.Error(t, err) +} + +func TestReadCatFileObject(t *testing.T) { + stream := "abc commit 5\nhello\n" + "def commit 0\n\n" + reader := bufio.NewReader(strings.NewReader(stream)) + + sha, content, err := readCatFileObject(reader) + require.NoError(t, err) + assert.Equal(t, "abc", sha) + assert.Equal(t, "hello", string(content)) + + sha, content, err = readCatFileObject(reader) + require.NoError(t, err) + assert.Equal(t, "def", sha) + assert.Empty(t, content) + + _, _, err = readCatFileObject(reader) + assert.ErrorIs(t, err, io.EOF) +} diff --git a/backend/plugins/gitextractor/parser/gitrunner.go b/backend/plugins/gitextractor/parser/gitrunner.go new file mode 100644 index 000000000..f3bb8e3d6 --- /dev/null +++ b/backend/plugins/gitextractor/parser/gitrunner.go @@ -0,0 +1,155 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "os/exec" + "strings" +) + +// gitRunner runs read-only git commands against one local repository. +// +// Output and Stream both run the git command in a way that never writes to the +// repository and never needs a working tree, so bare clones are supported. +type gitRunner struct { + dir string +} + +func newGitRunner(dir string) *gitRunner { + return &gitRunner{dir: dir} +} + +// Output runs git and returns stdout. Stderr is reported in the error. +func (r *gitRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + var stdout, stderr bytes.Buffer + cmd := r.command(ctx, args...) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + return nil, fmt.Errorf("git %s failed: %s", strings.Join(sanitizeArgs(args), " "), errorMessage(stderr.Bytes(), err)) + } + return stdout.Bytes(), nil +} + +// OutputLines runs git and returns its stdout lines with the trailing newline +// removed from each line. +func (r *gitRunner) OutputLines(ctx context.Context, args ...string) ([]string, error) { + out, err := r.Output(ctx, args...) + if err != nil { + return nil, err + } + return splitLines(out), nil +} + +// Stream runs git and returns its stdout. The caller must read to EOF and then +// call the returned wait function; closing the reader early kills git. +func (r *gitRunner) Stream(ctx context.Context, args ...string) (io.ReadCloser, func() error, error) { + cmd := r.command(ctx, args...) + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, err + } + stderr := &bytes.Buffer{} + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + return nil, nil, err + } + return stdout, func() error { + err := cmd.Wait() + if err == nil { + return nil + } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return fmt.Errorf("git %s failed: %s", strings.Join(sanitizeArgs(args), " "), errorMessage(stderr.Bytes(), err)) + }, nil +} + +// Pipe runs git with stdin and returns a reader over its stdout. +func (r *gitRunner) Pipe(ctx context.Context, stdin []byte, args ...string) (io.ReadCloser, func() error, error) { + cmd := r.command(ctx, args...) + cmd.Stdin = bytes.NewReader(stdin) + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, err + } + stderr := &bytes.Buffer{} + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + return nil, nil, err + } + return stdout, func() error { + err := cmd.Wait() + if err == nil { + return nil + } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return fmt.Errorf("git %s failed: %s", strings.Join(sanitizeArgs(args), " "), errorMessage(stderr.Bytes(), err)) + }, nil +} + +func (r *gitRunner) command(ctx context.Context, args ...string) *exec.Cmd { + fullArgs := append([]string{"-c", "core.quotePath=false", "-c", "gc.auto=0", "-c", "maintenance.auto=false"}, args...) + cmd := exec.CommandContext(ctx, "git", fullArgs...) + cmd.Dir = r.dir + cmd.Env = append(cmd.Environ(), + "GIT_OPTIONAL_LOCKS=0", + "GIT_NO_LAZY_FETCH=1", + "GIT_PAGER=cat", + "GIT_TERMINAL_PROMPT=0", + "LC_ALL=C", + ) + return cmd +} + +func errorMessage(stderr []byte, err error) string { + if msg := strings.TrimSpace(string(stderr)); msg != "" { + return msg + } + return err.Error() +} + +func splitLines(out []byte) []string { + text := strings.TrimSuffix(string(out), "\n") + if text == "" { + return nil + } + return strings.Split(text, "\n") +} + +func scanLines(reader io.Reader, fn func(line string) error) error { + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) + for scanner.Scan() { + if err := fn(scanner.Text()); err != nil { + return err + } + } + return scanner.Err() +} diff --git a/backend/plugins/gitextractor/parser/numstatparse.go b/backend/plugins/gitextractor/parser/numstatparse.go new file mode 100644 index 000000000..1d96dd6ae --- /dev/null +++ b/backend/plugins/gitextractor/parser/numstatparse.go @@ -0,0 +1,130 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "fmt" + "io" + "strconv" + "strings" +) + +const numstatRecordMarker = '\x01' + +type commitFileStat struct { + Path string + Additions int + Deletions int +} + +type commitStats struct { + Sha string + Files []commitFileStat +} + +// parseNumstatStream parses the output of +// +// git log --format=%x01%H --numstat --no-renames -z ... +// +// Each commit is introduced by a \x01<sha> segment; every following NUL +// terminated segment is "additions<TAB>deletions<TAB>path". Paths may contain +// tabs, spaces and newlines, but never NUL, so splitting on NUL is safe. +func parseNumstatStream(reader io.Reader, fn func(*commitStats) error) error { + buffered := bufio.NewReaderSize(reader, 256*1024) + + var current *commitStats + flush := func() error { + if current == nil { + return nil + } + if err := fn(current); err != nil { + return err + } + current = nil + return nil + } + + for { + segment, err := buffered.ReadString(0) + if err != nil && err != io.EOF { + return err + } + // ReadString returns data without the delimiter on EOF + segment = strings.TrimSuffix(segment, "\x00") + + switch { + case segment == "": + // nothing to do; covers a trailing NUL + case segment[0] == numstatRecordMarker: + if err := flush(); err != nil { + return err + } + current = &commitStats{Sha: segment[1:]} + default: + if current == nil { + return fmt.Errorf("numstat entry %q before any commit", segment) + } + entry := strings.TrimPrefix(segment, "\n") + stat, err := parseNumstatEntry(entry) + if err != nil { + return fmt.Errorf("commit %s: %w", current.Sha, err) + } + current.Files = append(current.Files, stat) + } + + if err == io.EOF { + break + } + } + return flush() +} + +func parseNumstatEntry(entry string) (commitFileStat, error) { + var stat commitFileStat + parts := strings.SplitN(entry, "\t", 3) + if len(parts) != 3 { + return stat, fmt.Errorf("unexpected numstat entry %q", entry) + } + stat.Path = parts[2] + + additions, err := parseNumstatCount(parts[0]) + if err != nil { + return stat, err + } + deletions, err := parseNumstatCount(parts[1]) + if err != nil { + return stat, err + } + stat.Additions = additions + stat.Deletions = deletions + return stat, nil +} + +// parseNumstatCount parses a numstat count. Binary files are reported as "-" +// and count as zero changed lines, just like libgit2 reports them. +func parseNumstatCount(value string) (int, error) { + if value == "-" { + return 0, nil + } + count, err := strconv.Atoi(value) + if err != nil { + return 0, fmt.Errorf("unexpected numstat count %q", value) + } + return count, nil +} diff --git a/backend/plugins/gitextractor/parser/parse_test.go b/backend/plugins/gitextractor/parser/parse_test.go new file mode 100644 index 000000000..23783c329 --- /dev/null +++ b/backend/plugins/gitextractor/parser/parse_test.go @@ -0,0 +1,243 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseNumstatStream(t *testing.T) { + stream := strings.Join([]string{ + "\x01sha1\x00", + "\n1\t0\ta.txt\x00", + "-\t-\tbin.dat\x00", + "\x01sha2\x00", + "\x01sha3\x00", + "\n2\t1\tdir with space/f.txt\x00", + "1\t0\tüñï.txt\x00", + "0\t0\tempty.txt\x00", + "1\t0\tnew\nline.txt\x00", + }, "") + + var commits []*commitStats + err := parseNumstatStream(strings.NewReader(stream), func(commit *commitStats) error { + commits = append(commits, commit) + return nil + }) + require.NoError(t, err) + + require.Len(t, commits, 3) + assert.Equal(t, "sha1", commits[0].Sha) + assert.Equal(t, []commitFileStat{ + {Path: "a.txt", Additions: 1}, + {Path: "bin.dat"}, + }, commits[0].Files) + assert.Equal(t, "sha2", commits[1].Sha) + assert.Empty(t, commits[1].Files) + assert.Equal(t, []commitFileStat{ + {Path: "dir with space/f.txt", Additions: 2, Deletions: 1}, + {Path: "üñï.txt", Additions: 1}, + {Path: "empty.txt"}, + {Path: "new\nline.txt", Additions: 1}, + }, commits[2].Files) +} + +func TestParseNumstatStreamRejectsBadInput(t *testing.T) { + err := parseNumstatStream(strings.NewReader("1\t0\ta.txt\x00"), func(*commitStats) error { return nil }) + assert.Error(t, err, "an entry without a commit marker is invalid") + + err = parseNumstatStream(strings.NewReader("\x01sha\x00\nx\t0\ta.txt\x00"), func(*commitStats) error { return nil }) + assert.Error(t, err, "a non-numeric count is invalid") +} + +func TestParsePatchStream(t *testing.T) { + patch := strings.Join([]string{ + "\x01c1", + "diff --git a/a.txt b/a.txt", + "new file mode 100644", + "index 0000000..1111111", + "--- /dev/null", + "+++ b/a.txt", + "@@ -0,0 +1,3 @@", + "+l1", + "+l2", + "+l3", + "\x01c2", + "diff --git a/a.txt b/a.txt", + "index 1111111..2222222 100644", + "--- a/a.txt", + "+++ b/a.txt", + "@@ -1,3 +1,4 @@ heading", + " l1", + "-l2", + "+L2", + " l3", + "+l4", + "\\ No newline at end of file", + "diff --git a/bin.dat b/bin.dat", + "index 3333333..4444444 100644", + "Binary files a/bin.dat and b/bin.dat differ", + "diff --git a/empty.txt b/empty.txt", + "deleted file mode 100644", + "index 5555555..0000000", + "--- a/empty.txt", + "+++ /dev/null", + "@@ -1 +0,0 @@", + "-x", + "diff --git a/mode.txt b/mode.txt", + "old mode 100644", + "new mode 100755", + "diff --git \"a/tab\\tname.txt\" \"b/tab\\tname.txt\"", + "index 6666666..7777777 100644", + "--- \"a/tab\\tname.txt\"", + "+++ \"b/tab\\tname.txt\"", + "@@ -1 +1 @@", + "-a", + "+b", + "", + }, "\n") + + var commits []*commitPatch + err := parsePatchStream(strings.NewReader(patch), func(commit *commitPatch) error { + commits = append(commits, commit) + return nil + }) + require.NoError(t, err) + + require.Len(t, commits, 2) + + c1 := commits[0] + assert.Equal(t, "c1", c1.Sha) + require.Len(t, c1.Files, 1) + assert.Equal(t, patchFile{ + OldPath: "a.txt", + NewPath: "a.txt", + Lines: []patchLine{ + {Origin: '+', OldLine: -1, NewLine: 1, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 2, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 3, HunkNum: 1}, + }, + }, c1.Files[0]) + + c2 := commits[1] + assert.Equal(t, "c2", c2.Sha) + require.Len(t, c2.Files, 5) + + assert.Equal(t, patchFile{ + OldPath: "a.txt", + NewPath: "a.txt", + Lines: []patchLine{ + {Origin: ' ', OldLine: 1, NewLine: 1, HunkNum: 1}, + {Origin: '-', OldLine: 2, NewLine: -1, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 2, HunkNum: 1}, + {Origin: ' ', OldLine: 3, NewLine: 3, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 4, HunkNum: 1}, + }, + }, c2.Files[0]) + + // binary file: both paths are known, but there are no line changes + assert.Equal(t, patchFile{OldPath: "bin.dat", NewPath: "bin.dat"}, c2.Files[1]) + + // deleted file: libgit2 reports the path on both sides + assert.Equal(t, patchFile{ + OldPath: "empty.txt", + NewPath: "empty.txt", + Lines: []patchLine{{Origin: '-', OldLine: 1, NewLine: -1, HunkNum: 1}}, + }, c2.Files[2]) + + // mode-only change + assert.Equal(t, patchFile{OldPath: "mode.txt", NewPath: "mode.txt"}, c2.Files[3]) + + // quoted path + assert.Equal(t, patchFile{ + OldPath: "tab\tname.txt", + NewPath: "tab\tname.txt", + Lines: []patchLine{ + {Origin: '-', OldLine: 1, NewLine: -1, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 1, HunkNum: 1}, + }, + }, c2.Files[4]) +} + +func TestParsePatchStreamMultipleHunksAndZeroStart(t *testing.T) { + patch := strings.Join([]string{ + "\x01c1", + "diff --git a/a.txt b/a.txt", + "--- a/a.txt", + "+++ b/a.txt", + "@@ -2,1 +2,1 @@", + "-old", + "+new", + "@@ -10,0 +11,2 @@", + "+x", + "+y", + "", + }, "\n") + + var commits []*commitPatch + require.NoError(t, parsePatchStream(strings.NewReader(patch), func(commit *commitPatch) error { + commits = append(commits, commit) + return nil + })) + require.Len(t, commits, 1) + assert.Equal(t, []patchLine{ + {Origin: '-', OldLine: 2, NewLine: -1, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 2, HunkNum: 1}, + {Origin: '+', OldLine: -1, NewLine: 11, HunkNum: 2}, + {Origin: '+', OldLine: -1, NewLine: 12, HunkNum: 2}, + }, commits[0].Files[0].Lines) +} + +func TestParseGitVersion(t *testing.T) { + for _, tc := range []struct { + output string + want int + }{ + {"git version 2.43.0\n", 2*10000 + 43*100}, + {"git version 2.31.0", 2*10000 + 31*100}, + {"git version 2.39.2.windows.1", 2*10000 + 39*100 + 2}, + {"git version 2.56.0", 2*10000 + 56*100}, + } { + got, err := parseGitVersion(tc.output) + if assert.NoError(t, err, tc.output) { + assert.Equal(t, tc.want, got, tc.output) + } + } + + _, err := parseGitVersion("not git") + assert.Error(t, err) +} + +func TestUnquoteGitPath(t *testing.T) { + for _, tc := range []struct { + in, want string + }{ + {`a/plain.txt`, `a/plain.txt`}, + {`"a/tab\tname.txt"`, "a/tab\tname.txt"}, + {`"a/quote\"name.txt"`, `a/quote"name.txt`}, + {`"a/back\\slash"`, `a/back\slash`}, + {`"a/\303\274ber.txt"`, "a/über.txt"}, + {`"a/new\nline.txt"`, "a/new\nline.txt"}, + } { + assert.Equal(t, tc.want, unquoteGitPath(tc.in), tc.in) + } +} diff --git a/backend/plugins/gitextractor/parser/patchparse.go b/backend/plugins/gitextractor/parser/patchparse.go new file mode 100644 index 000000000..e4d3c8dc8 --- /dev/null +++ b/backend/plugins/gitextractor/parser/patchparse.go @@ -0,0 +1,332 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "fmt" + "io" + "strconv" + "strings" +) + +const ( + patchChangeContext = ' ' + patchChangeAddition = '+' + patchChangeDeletion = '-' + + // patchRecordMarker introduces the commit sha in the `git log --format` + // output. A patch line can never start with it: every patch content line + // starts with ' ', '+' or '-'. + patchRecordMarker = "\x01" +) + +type patchLine struct { + Origin byte + OldLine int + NewLine int + HunkNum int +} + +type patchFile struct { + OldPath string + NewPath string + Lines []patchLine +} + +type commitPatch struct { + Sha string + Files []patchFile +} + +type patchParser struct { + commit *commitPatch + current *patchFile + + oldLine int + newLine int + hunkNum int + + inHunk bool +} + +// parsePatchStream parses `git log --format=%x01%H -p ...` output. +func parsePatchStream(reader io.Reader, fn func(*commitPatch) error) error { + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 0, 256*1024), 16*1024*1024) + + var parser *patchParser + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, patchRecordMarker) { + if err := flushPatch(parser, fn); err != nil { + return err + } + parser = &patchParser{commit: &commitPatch{Sha: line[len(patchRecordMarker):]}} + continue + } + if parser == nil { + continue + } + if err := parser.line(line); err != nil { + return err + } + } + if err := scanner.Err(); err != nil { + return err + } + return flushPatch(parser, fn) +} + +func flushPatch(parser *patchParser, fn func(*commitPatch) error) error { + if parser == nil || parser.commit == nil { + return nil + } + parser.endFile() + commit := parser.commit + parser.commit = nil + return fn(commit) +} + +func (p *patchParser) line(line string) error { + switch { + case strings.HasPrefix(line, "diff --git "): + p.endFile() + oldPath, newPath, err := parseDiffGitPaths(strings.TrimPrefix(line, "diff --git ")) + if err != nil { + return err + } + p.current = &patchFile{OldPath: oldPath, NewPath: newPath} + p.inHunk = false + return nil + case strings.HasPrefix(line, "--- "): + if p.current == nil { + p.current = &patchFile{} + } + p.current.OldPath = parsePatchPath(strings.TrimPrefix(line, "--- ")) + return nil + case strings.HasPrefix(line, "+++ "): + if p.current == nil { + p.current = &patchFile{} + } + p.current.NewPath = parsePatchPath(strings.TrimPrefix(line, "+++ ")) + return nil + case strings.HasPrefix(line, "@@ "): + return p.hunkHeader(line) + } + + if !p.inHunk { + return nil + } + + switch line[0] { + case patchChangeContext: + p.appendLine(patchChangeContext, p.oldLine, p.newLine) + p.oldLine++ + p.newLine++ + case patchChangeAddition: + p.appendLine(patchChangeAddition, -1, p.newLine) + p.newLine++ + case patchChangeDeletion: + p.appendLine(patchChangeDeletion, p.oldLine, -1) + p.oldLine++ + case '\\': + // "\ No newline at end of file" + } + return nil +} + +func (p *patchParser) appendLine(origin byte, oldLine, newLine int) { + if p.current == nil { + return + } + p.current.Lines = append(p.current.Lines, patchLine{ + Origin: origin, + OldLine: oldLine, + NewLine: newLine, + HunkNum: p.hunkNum, + }) +} + +func (p *patchParser) hunkHeader(line string) error { + // @@ -oldStart[,oldCount] +newStart[,newCount] @@ optional heading + end := strings.Index(line[3:], " @@") + if end < 0 { + return fmt.Errorf("unexpected hunk header %q", line) + } + fields := strings.SplitN(line[3:3+end], " ", 2) + if len(fields) != 2 { + return fmt.Errorf("unexpected hunk header %q", line) + } + oldStart, err := parseHunkRange(fields[0]) + if err != nil { + return fmt.Errorf("unexpected hunk header %q: %w", line, err) + } + newStart, err := parseHunkRange(fields[1]) + if err != nil { + return fmt.Errorf("unexpected hunk header %q: %w", line, err) + } + p.oldLine = oldStart + p.newLine = newStart + p.hunkNum++ + p.inHunk = true + return nil +} + +func parseHunkRange(value string) (int, error) { + value = strings.TrimPrefix(value, "-") + value = strings.TrimPrefix(value, "+") + if idx := strings.IndexByte(value, ','); idx >= 0 { + value = value[:idx] + } + start, err := strconv.Atoi(value) + if err != nil { + return 0, err + } + return start, nil +} + +func (p *patchParser) endFile() { + if p.current != nil { + // libgit2 reports the same path on both sides for additions and + // deletions, so mirror that here. + if p.current.OldPath == "" { + p.current.OldPath = p.current.NewPath + } + if p.current.NewPath == "" { + p.current.NewPath = p.current.OldPath + } + p.commit.Files = append(p.commit.Files, *p.current) + p.current = nil + } + p.inHunk = false + p.hunkNum = 0 +} + +// parsePatchPath strips the a/ or b/ prefix and unquotes git's C-style path +// quoting. /dev/null becomes the empty path. +func parsePatchPath(value string) string { + if value == "/dev/null" { + return "" + } + return stripGitPathPrefix(unquoteGitPath(value)) +} + +// parseDiffGitPaths extracts both paths from "a/<path> b/<path>". +// +// Without rename detection both sides are the same path, so finding the split +// point is unambiguous. Quoted paths are parsed as two quoted tokens. +func parseDiffGitPaths(value string) (string, string, error) { + if strings.HasPrefix(value, `"`) { + first, rest, err := readQuotedGitPath(value) + if err != nil { + return "", "", fmt.Errorf("invalid diff header %q: %w", value, err) + } + rest = strings.TrimPrefix(rest, " ") + second, _, err := readQuotedGitPath(rest) + if err != nil { + return "", "", fmt.Errorf("invalid diff header %q: %w", value, err) + } + return stripGitPathPrefix(first), stripGitPathPrefix(second), nil + } + for i := 2; i+3 <= len(value); i++ { + if value[i:i+3] != " b/" { + continue + } + oldPath := value[2:i] + newPath := value[i+3:] + if oldPath == newPath { + return oldPath, newPath, nil + } + } + return "", "", fmt.Errorf("invalid diff header %q", value) +} + +func stripGitPathPrefix(value string) string { + if len(value) > 2 && (value[0] == 'a' || value[0] == 'b') && value[1] == '/' { + return value[2:] + } + return value +} +func readQuotedGitPath(value string) (string, string, error) { + if !strings.HasPrefix(value, `"`) { + return "", value, fmt.Errorf("expected a quoted path") + } + for i := 1; i < len(value); i++ { + if value[i] == '\\' { + i++ + continue + } + if value[i] == '"' { + return unquoteGitPath(value[:i+1]), value[i+1:], nil + } + } + return "", value, fmt.Errorf("unterminated quoted path") +} + +// unquoteGitPath decodes git's C-style quoted paths, e.g. "a/\303\274ber". +func unquoteGitPath(value string) string { + if len(value) < 2 || value[0] != '"' || value[len(value)-1] != '"' { + return value + } + var out strings.Builder + for i := 1; i < len(value)-1; i++ { + char := value[i] + if char != '\\' { + out.WriteByte(char) + continue + } + i++ + if i >= len(value)-1 { + break + } + switch value[i] { + case 'a': + out.WriteByte('\a') + case 'b': + out.WriteByte('\b') + case 'f': + out.WriteByte('\f') + case 'n': + out.WriteByte('\n') + case 'r': + out.WriteByte('\r') + case 't': + out.WriteByte('\t') + case 'v': + out.WriteByte('\v') + case '\\', '"': + out.WriteByte(value[i]) + case '0', '1', '2', '3', '4', '5', '6', '7': + end := i + 3 + if end > len(value)-1 { + end = len(value) - 1 + } + code, err := strconv.ParseUint(value[i:end], 8, 8) + if err != nil { + out.WriteByte(value[i]) + continue + } + out.WriteByte(byte(code)) + i = end - 1 + default: + out.WriteByte(value[i]) + } + } + return out.String() +} diff --git a/backend/plugins/gitextractor/parser/repo_gitcli.go b/backend/plugins/gitextractor/parser/repo_gitcli.go new file mode 100644 index 000000000..4cf1fc0d1 --- /dev/null +++ b/backend/plugins/gitextractor/parser/repo_gitcli.go @@ -0,0 +1,753 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/apache/devlake/core/dal" + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/log" + "github.com/apache/devlake/core/models/domainlayer" + "github.com/apache/devlake/core/models/domainlayer/code" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/plugins/gitextractor/models" +) + +// minGitVersion is the first release with --diff-merges=first-parent, which is +// required to diff merge commits against their first parent only. +const minGitVersion = 2*10000 + 31 + +// commitBatchSize bounds how many commits are fetched and diffed at a time so +// that a repository with a long history does not need to be held in memory. +const commitBatchSize = 1024 + +var _ RepoCollector = (*GitcliRepoCollector)(nil) + +// GitcliRepoCollector collects git data with the git command line, so the +// server no longer needs libgit2 or a pure Go git implementation. +type GitcliRepoCollector struct { + id string + logger log.Logger + store models.Store + git *gitRunner + cleanup func() +} + +func NewGitcliRepoCollector(localDir string, repoId string, store models.Store, logger log.Logger) (*GitcliRepoCollector, errors.Error) { + git := newGitRunner(localDir) + if _, err := git.Output(context.Background(), "rev-parse", "--git-dir"); err != nil { + return nil, errors.Convert(err) + } + version, err := gitVersion(git) + if err != nil { + return nil, errors.Convert(err) + } + if version < minGitVersion { + return nil, errors.Default.New(fmt.Sprintf( + "git %s is not supported, gitextractor requires git 2.31 or newer", formatGitVersion(version))) + } + return &GitcliRepoCollector{ + id: repoId, + logger: logger, + store: store, + git: git, + }, nil +} + +func (r *GitcliRepoCollector) SetCleanUp(f func()) error { + if f != nil { + r.cleanup = f + } + return nil +} + +func (r *GitcliRepoCollector) Close(ctx context.Context) error { + defer func() { + if r.cleanup != nil { + r.cleanup() + } + }() + return r.store.Close() +} + +// CollectAll The main parser subtask +func (r *GitcliRepoCollector) CollectAll(subtaskCtx plugin.SubTaskContext) error { + subtaskCtx.SetProgress(0, -1) + if err := r.CollectTags(subtaskCtx); err != nil { + return err + } + if err := r.CollectBranches(subtaskCtx); err != nil { + return err + } + if err := r.CollectCommits(subtaskCtx); err != nil { + return err + } + if *subtaskCtx.GetData().(*GitExtractorTaskData).Options.SkipCommitStat { + return nil + } + return r.CollectDiffLine(subtaskCtx) +} + +// CountTags Count git tags subtask +func (r *GitcliRepoCollector) CountTags(ctx context.Context) (int, error) { + tags, err := r.tagRefs(ctx) + if err != nil { + return 0, err + } + return len(tags), nil +} + +// CountBranches count the number of branches in a git repo +func (r *GitcliRepoCollector) CountBranches(ctx context.Context) (int, error) { + branches, err := r.branchRefs(ctx) + if err != nil { + return 0, err + } + head, err := r.headRef(ctx) + if err != nil { + return 0, err + } + count := 0 + for _, branch := range branches { + if branch.refname != head { + count++ + } + } + return count, nil +} + +// CountCommits count the number of commits in a git repo +func (r *GitcliRepoCollector) CountCommits(ctx context.Context) (int, error) { + out, err := r.git.Output(ctx, "rev-list", "--all", "--count") + if err != nil { + return 0, err + } + count, err := strconv.Atoi(strings.TrimSpace(string(out))) + if err != nil { + return 0, fmt.Errorf("unexpected commit count %q", strings.TrimSpace(string(out))) + } + return count, nil +} + +// CollectTags Collect Tags data +func (r *GitcliRepoCollector) CollectTags(subtaskCtx plugin.SubTaskContext) error { + tags, err := r.tagRefs(subtaskCtx.GetContext()) + if err != nil { + return err + } + for _, tag := range tags { + commitSha := tag.peeledSha + if commitSha == "" { + commitSha = tag.sha + } + ref := &code.Ref{ + DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, tag.refname)}, + RepoId: r.id, + Name: tag.refname, + CommitSha: commitSha, + RefType: TAG, + } + if err := r.store.Refs(ref); err != nil { + return err + } + subtaskCtx.IncProgress(1) + } + return nil +} + +// CollectBranches Collect branch data +func (r *GitcliRepoCollector) CollectBranches(subtaskCtx plugin.SubTaskContext) error { + branches, err := r.branchRefs(subtaskCtx.GetContext()) + if err != nil { + return err + } + head, err := r.headRef(subtaskCtx.GetContext()) + if err != nil { + return err + } + for _, branch := range branches { + ref := &code.Ref{ + DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, branch.name)}, + RepoId: r.id, + Name: branch.name, + CommitSha: branch.sha, + RefType: BRANCH, + IsDefault: branch.refname == head, + } + if err := r.store.Refs(ref); err != nil { + return err + } + subtaskCtx.IncProgress(1) + } + return nil +} + +// CollectCommits Collect data from each commit, we can also get the diff line +func (r *GitcliRepoCollector) CollectCommits(subtaskCtx plugin.SubTaskContext) (err error) { + ctx := subtaskCtx.GetContext() + taskOpts := subtaskCtx.GetData().(*GitExtractorTaskData).Options + + shas, err := r.git.OutputLines(ctx, "rev-list", "--all") + if err != nil { + return err + } + if len(shas) == 0 { + return nil + } + present := make(map[string]struct{}, len(shas)) + for _, sha := range shas { + present[sha] = struct{}{} + } + shallow, err := r.shallowCommits(ctx) + if err != nil { + return err + } + componentMap, err := r.componentMap(subtaskCtx) + if err != nil { + return err + } + excluded := normalizeExtensions(taskOpts.ExcludeFileExtensions) + + for start := 0; start < len(shas); start += commitBatchSize { + end := start + commitBatchSize + if end > len(shas) { + end = len(shas) + } + batch := shas[start:end] + + var statsBySha map[string]*commitStats + if !*taskOpts.SkipCommitStat { + statsBySha, err = r.commitStats(ctx, batch) + if err != nil { + return err + } + } + commits, err := r.readCommits(ctx, batch) + if err != nil { + return err + } + for _, metadata := range commits { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + // A shallow boundary commit's parent is not in the object + // database. Diffing it would produce a full-tree diff and inflate + // the addition count, so it is skipped entirely. + if _, ok := shallow[metadata.Sha]; ok { + r.logger.Info("skip commit %s because it is a shallow boundary", metadata.Sha) + continue + } + // The same guard as above, for repositories where the boundary is + // not recorded in the shallow file. + if len(metadata.Parents) > 0 { + if _, ok := present[metadata.Parents[0]]; !ok { + r.logger.Info("skip commit %s because it has no parent commit", metadata.Sha) + continue + } + } + + commit := &code.Commit{ + Sha: metadata.Sha, + Message: metadata.Message, + AuthorName: metadata.Author.Name, + AuthorEmail: metadata.Author.Email, + AuthorId: metadata.Author.Email, + AuthoredDate: metadata.Author.When, + CommitterName: metadata.Committer.Name, + CommitterEmail: metadata.Committer.Email, + CommitterId: metadata.Committer.Email, + CommittedDate: metadata.Committer.When, + } + + var files []commitFileStat + if stats := statsBySha[metadata.Sha]; stats != nil { + files = stats.Files + } + for _, file := range files { + if isExcludedPath(file.Path, excluded) { + continue + } + commit.Additions += file.Additions + commit.Deletions += file.Deletions + } + + if err := r.storeParentCommits(metadata, present); err != nil { + return err + } + if err := r.store.Commits(commit); err != nil { + return err + } + if err := r.store.RepoCommits(&code.RepoCommit{RepoId: r.id, CommitSha: commit.Sha}); err != nil { + return err + } + + if !*taskOpts.SkipCommitFiles { + for _, file := range files { + if isExcludedPath(file.Path, excluded) { + continue + } + if err := r.storeCommitFile(commit.Sha, file, componentMap); err != nil { + return err + } + } + } + subtaskCtx.IncProgress(1) + } + } + return nil +} + +func (r *GitcliRepoCollector) storeParentCommits(commit *parsedCommit, present map[string]struct{}) errors.Error { + var parents []*code.CommitParent + for _, parentSha := range commit.Parents { + // Skip parents that are not in the object database, which happens for + // shallow clones, instead of storing a dangling reference. + if _, ok := present[parentSha]; !ok { + continue + } + parents = append(parents, &code.CommitParent{ + CommitSha: commit.Sha, + ParentCommitSha: parentSha, + }) + } + return r.store.CommitParents(parents) +} + +func (r *GitcliRepoCollector) storeCommitFile(commitSha string, file commitFileStat, componentMap map[string]*regexp.Regexp) errors.Error { + commitFile := &code.CommitFile{ + CommitSha: commitSha, + FilePath: file.Path, + Additions: file.Additions, + Deletions: file.Deletions, + } + commitFile.Id = genCommitFileId(commitFile.CommitSha, commitFile.FilePath) + + componentName := "Default" + for _, name := range sortedComponentNames(componentMap) { + if componentMap[name].MatchString(commitFile.FilePath) { + componentName = name + break + } + } + if err := r.store.CommitFileComponents(&code.CommitFileComponent{ + CommitFileId: commitFile.Id, + ComponentName: componentName, + }); err != nil { + return err + } + return r.store.CommitFiles(commitFile) +} + +// CollectDiffLine replays the first-parent history of HEAD and records the +// origin commit of every line, plus the per-line change rows. +func (r *GitcliRepoCollector) CollectDiffLine(subtaskCtx plugin.SubTaskContext) error { + ctx := subtaskCtx.GetContext() + + if _, err := r.git.Output(ctx, "rev-parse", "--verify", "HEAD"); err != nil { + // an empty repository has no HEAD and therefore no snapshot + r.logger.Info("skip diff line collection because the repository has no HEAD") + return nil + } + + reader, wait, err := r.git.Stream(ctx, + "log", "--first-parent", "--reverse", "--no-renames", "-p", "-U3", + "--format="+patchRecordMarker+"%H", + "--no-color", "--no-ext-diff", "--no-show-signature", "--no-notes", + "HEAD", + ) + if err != nil { + return err + } + + snapshot := make(map[string]*models.FileBlame) + parseErr := parsePatchStream(reader, func(patch *commitPatch) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + return r.storeSnapshotForCommit(patch, snapshot) + }) + waitErr := wait() + if parseErr != nil { + return parseErr + } + if waitErr != nil { + return waitErr + } + + db := subtaskCtx.GetDal() + if err := db.Delete(&code.RepoSnapshot{}, dal.Where("repo_id= ?", r.id)); err != nil { + return err + } + + files := make([]string, 0, len(snapshot)) + for file := range snapshot { + files = append(files, file) + } + sort.Strings(files) + for _, file := range files { + lineNo := 0 + for element := snapshot[file].Lines.Front(); element != nil; element = element.Next() { + lineNo++ + commitSha, ok := element.Value.(string) + if !ok { + // A placeholder without a commit should not happen, but do not + // fail the whole collection if it does. + r.logger.Warn(nil, "empty snapshot line for %s:%d", file, lineNo) + continue + } + if err := r.store.RepoSnapshot(&code.RepoSnapshot{ + RepoId: r.id, + CommitSha: commitSha, + FilePath: file, + LineNo: lineNo, + }); err != nil { + return err + } + } + } + r.logger.Info("collect snapshot finished") + return nil +} + +func (r *GitcliRepoCollector) storeSnapshotForCommit(patch *commitPatch, snapshot map[string]*models.FileBlame) error { + pending := &snapshotChanges{} + lastFile := "" + + for _, file := range patch.Files { + if _, ok := snapshot[file.OldPath]; !ok { + fileBlame, err := models.NewFileBlame() + if err != nil { + return err + } + snapshot[file.OldPath] = fileBlame + } + if lastFile == "" { + lastFile = file.NewPath + } else if lastFile != file.NewPath { + applySnapshotChanges(snapshot, lastFile, patch.Sha, pending) + pending = &snapshotChanges{} + lastFile = file.NewPath + } + + for _, line := range file.Lines { + change := &code.CommitLineChange{ + CommitSha: patch.Sha, + ChangedType: lineChangeType(line.Origin), + LineNoNew: line.NewLine, + LineNoOld: line.OldLine, + OldFilePath: file.OldPath, + NewFilePath: file.NewPath, + HunkNum: line.HunkNum, + Id: fmt.Sprintf("%s:%s:%d:%d", patch.Sha, file.NewPath, line.OldLine, line.NewLine), + } + switch line.Origin { + case patchChangeAddition: + pending.added = append(pending.added, line) + case patchChangeDeletion: + if blame := snapshot[file.OldPath]; blame != nil { + if element := blame.Find(line.OldLine); element != nil && element.Value != nil { + change.PrevCommit = element.Value.(string) + } + } + pending.deleted = append(pending.deleted, line) + } + if err := r.store.CommitLineChange(change); err != nil { + return err + } + } + } + applySnapshotChanges(snapshot, lastFile, patch.Sha, pending) + return nil +} + +type snapshotChanges struct { + deleted []patchLine + added []patchLine +} + +// applySnapshotChanges mirrors the line bookkeeping of the libgit2 collector: +// deletions remove the old line numbers and additions insert the commit sha at +// the new line numbers. +func applySnapshotChanges(snapshot map[string]*models.FileBlame, filePath, commitSha string, changes *snapshotChanges) { + fileBlame, ok := snapshot[filePath] + if !ok || fileBlame == nil || (len(changes.deleted) == 0 && len(changes.added) == 0) { + return + } + sort.SliceStable(changes.deleted, func(i, j int) bool { + return changes.deleted[i].OldLine > changes.deleted[j].OldLine + }) + for _, line := range changes.deleted { + fileBlame.RemoveLine(line.OldLine) + } + for _, line := range changes.added { + fileBlame.AddLine(line.NewLine, commitSha) + } +} + +func lineChangeType(origin byte) string { + switch origin { + case patchChangeAddition: + return "Addition" + case patchChangeDeletion: + return "Deletion" + default: + return "Context" + } +} + +// readCommits returns the commit objects of the batch in the order of the batch. +func (r *GitcliRepoCollector) readCommits(ctx context.Context, shas []string) ([]*parsedCommit, error) { + stdin := []byte(strings.Join(shas, "\n") + "\n") + reader, wait, err := r.git.Pipe(ctx, stdin, "cat-file", "--batch") + if err != nil { + return nil, err + } + buffered := bufio.NewReaderSize(reader, 256*1024) + commits := make([]*parsedCommit, 0, len(shas)) + for { + sha, content, readErr := readCatFileObject(buffered) + if readErr == io.EOF { + break + } + if readErr != nil { + _ = wait() + return nil, readErr + } + commit, parseErr := parseCommitObject(sha, content) + if parseErr != nil { + _ = wait() + return nil, parseErr + } + commits = append(commits, commit) + } + if err := wait(); err != nil { + return nil, err + } + return commits, nil +} + +// commitStats returns the per-file line counts of the batch. +func (r *GitcliRepoCollector) commitStats(ctx context.Context, shas []string) (map[string]*commitStats, error) { + stdin := []byte(strings.Join(shas, "\n") + "\n") + reader, wait, err := r.git.Pipe(ctx, stdin, + "log", "--no-walk=unsorted", "--stdin", "--root", "--diff-merges=first-parent", + "--numstat", "--no-renames", "-z", "--format="+patchRecordMarker+"%H", + "--no-color", "--no-ext-diff", "--no-show-signature", "--no-notes", + ) + if err != nil { + return nil, err + } + statsBySha := make(map[string]*commitStats, len(shas)) + err = parseNumstatStream(reader, func(stats *commitStats) error { + statsBySha[stats.Sha] = stats + return nil + }) + waitErr := wait() + if err != nil { + return nil, err + } + if waitErr != nil { + return nil, waitErr + } + return statsBySha, nil +} + +// shallowCommits returns the commits recorded as shallow boundaries. +func (r *GitcliRepoCollector) shallowCommits(ctx context.Context) (map[string]struct{}, error) { + shallow := make(map[string]struct{}) + out, err := r.git.Output(ctx, "rev-parse", "--git-path", "shallow") + if err != nil { + return shallow, err + } + content, err := os.ReadFile(strings.TrimSpace(string(out))) + if os.IsNotExist(err) { + return shallow, nil + } + if err != nil { + return shallow, err + } + for _, line := range strings.Split(string(content), "\n") { + if sha := strings.TrimSpace(line); sha != "" { + shallow[sha] = struct{}{} + } + } + return shallow, nil +} + +type tagRef struct { + refname string + sha string + peeledSha string +} + +func (r *GitcliRepoCollector) tagRefs(ctx context.Context) ([]tagRef, error) { + lines, err := r.git.OutputLines(ctx, "for-each-ref", + "--format=%(refname)%00%(objectname)%00%(*objectname)", + "refs/tags") + if err != nil { + return nil, err + } + tags := make([]tagRef, 0, len(lines)) + for _, line := range lines { + fields := strings.SplitN(line, "\x00", 3) + if len(fields) != 3 { + return nil, fmt.Errorf("unexpected tag ref %q", line) + } + tags = append(tags, tagRef{refname: fields[0], sha: fields[1], peeledSha: fields[2]}) + } + return tags, nil +} + +type branchRef struct { + refname string + name string + sha string +} + +func (r *GitcliRepoCollector) branchRefs(ctx context.Context) ([]branchRef, error) { + lines, err := r.git.OutputLines(ctx, "for-each-ref", + "--format=%(refname)%00%(objectname)", + "refs/heads", "refs/remotes") + if err != nil { + return nil, err + } + branches := make([]branchRef, 0, len(lines)) + for _, line := range lines { + fields := strings.SplitN(line, "\x00", 2) + if len(fields) != 2 { + return nil, fmt.Errorf("unexpected branch ref %q", line) + } + name := strings.TrimPrefix(fields[0], "refs/heads/") + name = strings.TrimPrefix(name, "refs/remotes/") + branches = append(branches, branchRef{refname: fields[0], name: name, sha: fields[1]}) + } + return branches, nil +} + +func (r *GitcliRepoCollector) headRef(ctx context.Context) (string, error) { + out, err := r.git.Output(ctx, "symbolic-ref", "-q", "HEAD") + if err != nil { + // a detached HEAD has no default branch + return "", nil + } + return strings.TrimSpace(string(out)), nil +} + +func (r *GitcliRepoCollector) componentMap(subtaskCtx plugin.SubTaskContext) (map[string]*regexp.Regexp, error) { + db := subtaskCtx.GetDal() + components := make([]code.Component, 0) + if err := db.All(&components, dal.From(components), dal.Where("repo_id= ?", r.id)); err != nil { + return nil, err + } + componentMap := make(map[string]*regexp.Regexp, len(components)) + for _, component := range components { + componentMap[component.Name] = regexp.MustCompile(component.PathRegex) + } + return componentMap, nil +} + +func sortedComponentNames(componentMap map[string]*regexp.Regexp) []string { + names := make([]string, 0, len(componentMap)) + for name := range componentMap { + names = append(names, name) + } + sort.Strings(names) + return names +} + +func normalizeExtensions(extensions []string) map[string]struct{} { + excluded := make(map[string]struct{}, len(extensions)) + for _, extension := range extensions { + value := strings.ToLower(strings.TrimSpace(extension)) + if value == "" { + continue + } + excluded[value] = struct{}{} + } + return excluded +} + +func isExcludedPath(path string, excluded map[string]struct{}) bool { + if len(excluded) == 0 { + return false + } + lower := strings.ToLower(path) + for extension := range excluded { + if strings.HasSuffix(lower, extension) { + return true + } + } + return false +} + +// gitVersion returns the version of git as major*10000 + minor*100 + patch. +func gitVersion(git *gitRunner) (int, error) { + out, err := git.Output(context.Background(), "version") + if err != nil { + return 0, err + } + return parseGitVersion(string(out)) +} + +func parseGitVersion(output string) (int, error) { + // e.g. "git version 2.43.0", "git version 2.39.2.windows.1" + fields := strings.Fields(output) + if len(fields) < 3 || fields[0] != "git" || fields[1] != "version" { + return 0, fmt.Errorf("unexpected git version output %q", strings.TrimSpace(output)) + } + parts := strings.SplitN(fields[2], ".", 3) + if len(parts) < 2 { + return 0, fmt.Errorf("unexpected git version %q", fields[2]) + } + major, err := strconv.Atoi(parts[0]) + if err != nil { + return 0, fmt.Errorf("unexpected git version %q", fields[2]) + } + minor, err := strconv.Atoi(parts[1]) + if err != nil { + return 0, fmt.Errorf("unexpected git version %q", fields[2]) + } + patch := 0 + if len(parts) == 3 { + if idx := strings.IndexFunc(parts[2], func(r rune) bool { return r < '0' || r > '9' }); idx >= 0 { + parts[2] = parts[2][:idx] + } + if parts[2] != "" { + patch, _ = strconv.Atoi(parts[2]) + } + } + return major*10000 + minor*100 + patch, nil +} + +func formatGitVersion(version int) string { + return fmt.Sprintf("%d.%d.%d", version/10000, version%10000/100, version%100) +} diff --git a/backend/plugins/gitextractor/parser/taskdata.go b/backend/plugins/gitextractor/parser/taskdata.go index bdfdbd2ae..a8bb4510d 100644 --- a/backend/plugins/gitextractor/parser/taskdata.go +++ b/backend/plugins/gitextractor/parser/taskdata.go @@ -49,4 +49,8 @@ type GitExtractorOptions struct { PluginName string `json:"pluginName" mapstructure:"pluginName,omitempty"` // Configured by upstream plugin (e.g., GitLab) to exclude file extensions from commit stats ExcludeFileExtensions []string `json:"excludeFileExtensions" mapstructure:"excludeFileExtensions"` + // UseGitCli selects the git command line collector. It is a temporary + // migration switch and will be removed once the library based collectors + // are gone. + UseGitCli bool `json:"useGitCli,omitempty" mapstructure:"useGitCli"` } diff --git a/backend/plugins/gitextractor/tasks/repo_cloner.go b/backend/plugins/gitextractor/tasks/repo_cloner.go index 76013e28e..63e58527b 100644 --- a/backend/plugins/gitextractor/tasks/repo_cloner.go +++ b/backend/plugins/gitextractor/tasks/repo_cloner.go @@ -71,7 +71,9 @@ func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error { } // We have done comparison experiments for git2go and go-git, and the results show that git2go has better performance. var repoCollector parser.RepoCollector - if *taskData.Options.UseGoGit { + if taskData.Options.UseGitCli { + repoCollector, err = parser.NewGitcliRepoCollector(localDir, op.RepoId, storage, logger) + } else if *taskData.Options.UseGoGit { repoCollector, err = parser.NewGogitRepoCollector(localDir, op.RepoId, storage, logger) } else { repoCollector, err = parser.NewLibgit2RepoCollector(localDir, op.RepoId, storage, logger)
