This is an automated email from the ASF dual-hosted git repository. klesh pushed a commit to branch feat/gitextractor-git-cli in repository https://gitbox.apache.org/repos/asf/devlake.git
commit 71a8c698882ccaf6f24c57b55f51cf3ee7b82569 Author: Klesh Wong <[email protected]> AuthorDate: Sun Nov 1 13:18:17 2026 +0800 refactor(gitextractor)!: remove libgit2 and go-git Drop the libgit2 (git2go) and go-git collectors now that the git CLI collector passes the contract suite. The backend has no import "C" left and builds with CGO_ENABLED=0. - delete repo_libgit2.go, repo_gogit.go and the git2go sort adapter - drop the useGoGit option/flag and the temporary gitcli switch - remove the go-git and git2go modules - remove install-libgit2.sh, the libgit2 build/copy steps from the Dockerfiles and devcontainer, and the CI install steps - drop the git2go dependabot ignore and USE_GO_GIT_IN_GIT_EXTRACTOR - update the gitextractor README, test docs and agent notes Behaviour changes inherited from the new collector: only commits reachable from refs are collected (dangling ODB objects are ignored), a component row is written for every commit file, and shallow boundary commits are skipped. --- .devcontainer/Dockerfile | 5 +- .github/dependabot.yml | 3 - .github/workflows/golangci-lint.yml | 2 - .github/workflows/test-e2e.yml | 4 - .github/workflows/test.yml | 2 - .gitignore | 1 - AGENTS.md | 2 +- backend/DevelopmentManual.md | 2 +- backend/Dockerfile | 75 +-- backend/Dockerfile.local | 27 +- backend/go.mod | 18 - backend/go.sum | 65 -- backend/plugins/gitextractor/README.md | 5 +- backend/plugins/gitextractor/gitextractor.go | 4 - backend/plugins/gitextractor/impl/impl.go | 5 - backend/plugins/gitextractor/models/diffline.go | 39 -- .../plugins/gitextractor/parser/clone_gitcli.go | 8 +- .../gitextractor/parser/clone_gitcli_test.go | 2 +- .../gitextractor/parser/collector_contract_test.go | 252 +++----- .../gitextractor/parser/commitparse_test.go | 2 +- .../plugins/gitextractor/parser/numstatparse.go | 2 +- backend/plugins/gitextractor/parser/parse_test.go | 20 +- backend/plugins/gitextractor/parser/patchparse.go | 4 +- backend/plugins/gitextractor/parser/repo_gitcli.go | 33 +- backend/plugins/gitextractor/parser/repo_gogit.go | 626 ------------------ .../plugins/gitextractor/parser/repo_libgit2.go | 698 --------------------- backend/plugins/gitextractor/parser/taskdata.go | 5 - backend/plugins/gitextractor/tasks/repo_cloner.go | 10 +- backend/plugins/kiro/impl/table_info_test.go | 7 +- backend/python/DevelopmentSetup.md | 2 +- backend/scripts/install-libgit2.sh | 126 ---- backend/test/Readme.md | 11 +- devops/docker/lake-builder/Dockerfile | 39 +- env.example | 3 - 34 files changed, 152 insertions(+), 1957 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b7099ace0..e79582508 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -15,10 +15,7 @@ FROM mcr.microsoft.com/devcontainers/go:2-1.27-bookworm -ENV PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib:/usr/local/lib/pkgconfig -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib ENV DEVLAKE_PLUGINS=argocd,bamboo,bitbucket,circleci,customize,dora,gitextractor,github,github_graphql,gitlab,issue_trace,jenkins,jira,org,pagerduty,refdiff,slack,sonarqube,trello,webhook RUN apt-get update -y -RUN apt-get install pkg-config python3-dev default-libmysqlclient-dev build-essential libpq-dev cmake -y -RUN git clone -b v1.5.0 https://github.com/libgit2/libgit2.git && cd libgit2 && mkdir build && cd build && cmake .. && make && make install +RUN apt-get install python3-dev default-libmysqlclient-dev build-essential libpq-dev -y diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 41f2b2a72..aaf59c2d5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -58,9 +58,6 @@ updates: # The Go toolchain directive is coupled to the CI bootstrap version and # to the golang base images; it is raised deliberately, never by a bot. - dependency-name: go - # git2go is ABI-coupled to the libgit2 version installed in the builder - # image. A mismatch fails at link time, so this is always manual. - - dependency-name: github.com/libgit2/git2go/* # swag v2 is still a release candidate and requires regenerating every # swagger annotation. Tracked as an externally gated upgrade. - dependency-name: github.com/swaggo/swag diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 0df4d8308..a21ca46c3 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -32,8 +32,6 @@ jobs: - uses: actions/checkout@v7 - name: Set git config run: git config --global --add safe.directory $(pwd) - - name: Install libgit2 1.5 for git2go v34 - run: backend/scripts/install-libgit2.sh - name: Install Go 1.27.3 run: backend/scripts/install-go.sh - name: Install mockery 3.8.1 diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 3499df44e..1512e7273 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -50,8 +50,6 @@ jobs: - name: Checkout code uses: actions/checkout@v7 - run: git config --global --add safe.directory $(pwd) - - name: Install libgit2 1.5 for git2go v34 - run: backend/scripts/install-libgit2.sh - name: Install Go 1.27.3 run: backend/scripts/install-go.sh - name: Build Python @@ -94,8 +92,6 @@ jobs: - name: Checkout code uses: actions/checkout@v7 - run: git config --global --add safe.directory $(pwd) - - name: Install libgit2 1.5 for git2go v34 - run: backend/scripts/install-libgit2.sh - name: Install Go 1.27.3 run: backend/scripts/install-go.sh - name: Build Python diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 86e2ea95e..6c0ac0707 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,8 +47,6 @@ jobs: uses: actions/checkout@v7 - name: Set git config run: git config --global --add safe.directory $(pwd) - - name: Install libgit2 1.5 for git2go v34 - run: backend/scripts/install-libgit2.sh - name: Install Go 1.27.3 run: backend/scripts/install-go.sh - name: Check go.mod is tidy without generated mocks diff --git a/.gitignore b/.gitignore index 0877343e6..7097c0187 100644 --- a/.gitignore +++ b/.gitignore @@ -147,7 +147,6 @@ NOTES.md main vendor bin -libgit2 .air.toml # Playwright CLI snapshots/logs (local dev) diff --git a/AGENTS.md b/AGENTS.md index b896b7af6..f558e08db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -129,4 +129,4 @@ Located in `backend/python/plugins/`. Use Poetry for dependencies. See [backend/ - Forgetting to add models to `GetTablesInfo()` fails `plugins/table_info_test.go` - Migration scripts must be added to `All()` in `register.go` - API changes require running `make swag` to update Swagger docs -- Python plugins require `libgit2` for gitextractor functionality +- `gitextractor` shells out to the `git` CLI; no git library or cgo is required diff --git a/backend/DevelopmentManual.md b/backend/DevelopmentManual.md index 26208d55f..96ba60958 100644 --- a/backend/DevelopmentManual.md +++ b/backend/DevelopmentManual.md @@ -161,7 +161,7 @@ func TestDevlakeServer(t *testing.T) { Plugins: []plugin.PluginMeta{ // list of plugins to make available to the server. Just create an empty instance of your intended plugin structs (conventionally declared in their respective `impl.go`) github.Github{}, githubGraphql.GithubGraphql{}, - gitextractor.GitExtractor{}, // this requires that your host machine has libgit2 installed, otherwise you'll get compilation errors + gitextractor.GitExtractor{}, gitlab.Gitlab(""), jenkins.Jenkins{}, webhook.Webhook{}, diff --git a/backend/Dockerfile b/backend/Dockerfile index 877a1ac7b..9ed02234a 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -23,14 +23,6 @@ #While incubation status is not necessarily a reflection of the completeness or stability of the code, #it does indicate that the project has yet to be fully endorsed by the ASF. -FROM --platform=linux/amd64 debian:bookworm as debian-amd64 -RUN apt-get update -RUN apt-get install -y libssh2-1-dev libssl-dev zlib1g-dev - -FROM --platform=linux/arm64 debian:bookworm as debian-arm64 -RUN apt-get update -RUN apt-get install -y libssh2-1-dev libssl-dev zlib1g-dev - FROM --platform=$BUILDPLATFORM golang:1.27-bookworm as builder # docker build --build-arg GOPROXY=https://goproxy.io,direct -t mericodev/lake . @@ -40,47 +32,11 @@ ARG HTTP_PROXY= ARG HTTPS_PROXY= RUN apt-get update -RUN apt-get install -y gcc binutils libfindbin-libs-perl cmake libssh2-1-dev libssl-dev zlib1g-dev - -RUN if [ "$(arch)" != "aarch64" ] ; then \ - apt-get install -y gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu ; \ - fi -RUN if [ "$(arch)" != "x86_64" ] ; then \ - apt-get install -y gcc-x86-64-linux-gnu binutils-x86-64-linux-gnu ; \ - fi +RUN apt-get install -y ca-certificates git RUN go install github.com/vektra/mockery/[email protected] RUN go install github.com/swaggo/swag/cmd/[email protected] -COPY --from=debian-amd64 /usr/include /rootfs-amd64/usr/include -COPY --from=debian-amd64 /usr/lib/x86_64-linux-gnu /rootfs-amd64/usr/lib/x86_64-linux-gnu -COPY --from=debian-amd64 /lib/x86_64-linux-gnu /rootfs-amd64/lib/x86_64-linux-gnu - -COPY --from=debian-arm64 /usr/include /rootfs-arm64/usr/include -COPY --from=debian-arm64 /usr/lib/aarch64-linux-gnu /rootfs-arm64/usr/lib/aarch64-linux-gnu -COPY --from=debian-arm64 /lib/aarch64-linux-gnu /rootfs-arm64/lib/aarch64-linux-gnu - - -# set -e: otherwise a build failure for one arch is silently skipped and the -# other arch's libgit2 is never installed under /usr/local/deps. -RUN set -e; \ - for arch in aarch64 x86_64 ; do \ - mkdir -p /tmp/build/${arch} && cd /tmp/build/${arch} && \ - wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.5.0.tar.gz -O - | tar -xz && \ - cd libgit2-1.5.0 && \ - mkdir build && cd build && \ - if [ "$arch" = "aarch64" ] ; then \ - cmake .. -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc \ - -DBUILD_SHARED_LIBS=ON -DUSE_BUNDLED_ZLIB=ON -DCMAKE_SYSROOT=/rootfs-arm64 \ - -DCMAKE_INSTALL_PREFIX=/usr/local/deps/${arch} ; \ - elif [ "$arch" = "x86_64" ] ; then \ - cmake .. -DCMAKE_C_COMPILER=x86_64-linux-gnu-gcc \ - -DBUILD_SHARED_LIBS=ON -DUSE_BUNDLED_ZLIB=ON -DCMAKE_SYSROOT=/rootfs-amd64 \ - -DCMAKE_INSTALL_PREFIX=/usr/local/deps/${arch} ; \ - fi && \ - make -j install ; \ - done - FROM builder as build @@ -95,35 +51,20 @@ ARG GO_PLUGINS= RUN --mount=type=cache,target=/root/.cache/go-build \ if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \ - ln -s /usr/local/deps/aarch64 /usr/local/deps/target && \ - export CC=aarch64-linux-gnu-gcc && \ - export GOARCH=arm64 && \ - export CGO_LDFLAGS="-Wl,-rpath-link,/rootfs-arm64/usr/lib/aarch64-linux-gnu -Wl,-rpath-link,/rootfs-arm64/lib/aarch64-linux-gnu" ; \ + export GOARCH=arm64 ; \ else \ - ln -s /usr/local/deps/x86_64 /usr/local/deps/target && \ - export CC=x86_64-linux-gnu-gcc && \ - export GOARCH=amd64 && \ - export CGO_LDFLAGS="-Wl,-rpath-link,/rootfs-amd64/usr/lib/x86_64-linux-gnu -Wl,-rpath-link,/rootfs-amd64/lib/x86_64-linux-gnu" ; \ + export GOARCH=amd64 ; \ fi && \ - export PKG_CONFIG_PATH=/usr/local/deps/target/lib/pkgconfig && \ - export CGO_ENABLED=1 &&\ + export CGO_ENABLED=0 && \ DEVLAKE_PLUGINS="$GO_PLUGINS" make all -# remove symlink in lib, we will recreate in final image -RUN cd /usr/local/deps/target/lib && \ - for file in *.so* ; do \ - if [ -L $file ] ; then \ - unlink $file ; \ - fi \ - done - FROM python:3.11-slim-bookworm as base # apt-get upgrade picks up Debian security fixes that the base tag does not contain yet RUN apt-get update && \ apt-get upgrade -y && \ - apt-get install -y python3-dev python3-pip tar pkg-config curl libssh2-1 zlib1g libffi-dev default-libmysqlclient-dev libpq-dev tini git openssh-client corkscrew && \ + apt-get install -y python3-dev python3-pip tar pkg-config curl zlib1g libffi-dev default-libmysqlclient-dev libpq-dev tini git openssh-client corkscrew && \ apt-get clean && \ rm -fr /usr/share/doc/* \ /usr/share/info/* \ @@ -166,12 +107,6 @@ RUN /app/python/build.sh FROM base as devlake-base ARG DEBUG= -# libraries -ENV LD_LIBRARY_PATH=/app/libs -RUN mkdir -p /app/libs -COPY --from=build /usr/local/deps/target/lib/*.so* /app/libs -RUN ldconfig -vn /app/libs - # apps # The single binary already contains every plugin; the .so files that # `make build-plugin` produces stay in the builder stage and are never copied. diff --git a/backend/Dockerfile.local b/backend/Dockerfile.local index e2046db3b..bd9a16cf2 100644 --- a/backend/Dockerfile.local +++ b/backend/Dockerfile.local @@ -31,21 +31,7 @@ ARG HTTPS_PROXY= RUN apt-get update && apt-get install -y \ gcc \ - binutils \ - libfindbin-libs-perl \ - cmake \ - libssh2-1-dev \ - libssl-dev \ - zlib1g-dev - -# Install libgit2 for native platform only -RUN mkdir -p /tmp/build && cd /tmp/build && \ - wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.5.0.tar.gz -O - | tar -xz && \ - cd libgit2-1.5.0 && \ - mkdir build && cd build && \ - cmake .. -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local && \ - make -j$(nproc) install && \ - ldconfig + ca-certificates RUN go install github.com/vektra/mockery/[email protected] RUN go install github.com/swaggo/swag/cmd/[email protected] @@ -82,21 +68,14 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ FROM debian:bookworm-slim RUN apt-get update && apt-get install -y \ - libssh2-1 \ - libssl3 \ ca-certificates \ - # gitextractor shells out to the git CLI unless - # USE_GO_GIT_IN_GIT_EXTRACTOR is set, so the binary must be present or every - # clone fails with "git: executable file not found in $PATH". + # gitextractor shells out to the git CLI, so the binary must be present or + # every clone fails with "git: executable file not found in $PATH". # debian:bookworm-slim does not ship it; the official image inherits it from # its python base. git \ && rm -rf /var/lib/apt/lists/* -# Copy libgit2 -COPY --from=builder /usr/local/lib/libgit2.so* /usr/local/lib/ -RUN ldconfig - WORKDIR /app COPY --from=builder /app/bin/lake ./bin/lake diff --git a/backend/go.mod b/backend/go.mod index f1f382839..fa0b0cbc5 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -13,7 +13,6 @@ require ( github.com/coreos/go-oidc/v3 v3.21.0 // github.com/gin-contrib/cors v1.6.0 github.com/gin-gonic/gin v1.12.0 - github.com/go-git/go-git/v5 v5.19.3 github.com/go-playground/validator/v10 v10.30.5 github.com/go-sql-driver/mysql v1.10.1 github.com/go-viper/mapstructure/v2 v2.5.0 @@ -21,7 +20,6 @@ require ( github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.11.0 - github.com/libgit2/git2go/v34 v34.0.0 github.com/magiconair/properties v1.18.12 github.com/merico-ai/graphql v0.0.0-20260206020408-b7fd267bcfac github.com/panjf2000/ants/v2 v2.12.1 @@ -48,11 +46,8 @@ require ( ) require ( - dario.cat/mergo v1.0.0 // indirect filippo.io/edwards25519 v1.2.0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/ProtonMail/go-crypto v1.1.6 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.24 // indirect @@ -72,19 +67,14 @@ require ( github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect - github.com/cloudflare/circl v1.6.3 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect - github.com/cyphar/filepath-securejoin v0.6.1 // indirect - github.com/emirpasic/gods v1.18.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.15 // indirect github.com/getsentry/sentry-go v0.46.0 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-errors/errors v1.4.2 // indirect - github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.9.2 // indirect github.com/go-jose/go-jose/v4 v4.1.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect @@ -95,17 +85,14 @@ require ( github.com/goccy/go-json v0.10.5 // indirect github.com/goccy/go-yaml v1.19.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect @@ -115,13 +102,10 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/pjbgf/sha1cd v0.7.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.1 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect - github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect - github.com/skeema/knownhosts v1.3.1 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/pflag v1.0.10 // indirect @@ -131,7 +115,6 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect - github.com/xanzy/ssh-agent v0.3.3 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/arch v0.22.0 // indirect @@ -141,6 +124,5 @@ require ( golang.org/x/text v0.42.0 // indirect golang.org/x/tools v0.51.0 // indirect google.golang.org/protobuf v1.36.12 // indirect - gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/backend/go.sum b/backend/go.sum index c6a7886bf..7e7964921 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1,22 +1,11 @@ -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= -github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go-v2 v1.47.4 h1:bTvk6UdSSSvaQsfLVj1JRDEEqj/PDRwGf1KanaRzau0= github.com/aws/aws-sdk-go-v2 v1.47.4/go.mod h1:XkrjAZvr705Nytv6PkPUhoSB2W9+vVHBp5oAWpkh8Ak= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.24 h1:t7FxqJ5KHDeKceyD4HtGaG6z7hMSeVGjs5/4c+L5ArM= @@ -63,8 +52,6 @@ github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiD github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/chainguard-dev/git-urls v1.0.2 h1:pSpT7ifrpc5X55n4aTTm7FFUE+ZQHKiqpiwNkJrVcKQ= github.com/chainguard-dev/git-urls v1.0.2/go.mod h1:rbGgj10OS7UgZlbzdUQIQpT0k/D4+An04HJY7Ol+Y/o= -github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= -github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/cockroachdb/errors v1.14.0 h1:EfdVEJpN3z8rPMo43Yit59LxoiIa470fSXpZXuEs+ZI= @@ -77,15 +64,9 @@ github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBq github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= -github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= -github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= -github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= -github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= @@ -100,18 +81,8 @@ github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= -github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= -github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.9.2 h1:OXFSRyz4g20upsGDJgQG9Bak1l/ZEv8GHVYB52O71sE= -github.com/go-git/go-billy/v5 v5.9.2/go.mod h1:ExsU+jcGwXTBOnyilvAnEM1wug1IxHr4yP2ZXsNRtV0= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.19.3 h1:qHttbZ+Am7wEjdTpR1XMQ4rl42FK9SIXzZ4o9x7s6M4= -github.com/go-git/go-git/v5 v5.19.3/go.mod h1:Ye4C8dVigkqrv9UsB4NMdSLV4yAIkLiIhW61gcOyhl4= github.com/go-jose/go-jose/v4 v4.1.6 h1:sNRkftxFJxwsILiQukqp5z83QBSmntVwmgRY7AqBDO4= github.com/go-jose/go-jose/v4 v4.1.6/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -150,13 +121,9 @@ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0kt github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= -github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= -github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -169,8 +136,6 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -179,8 +144,6 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= -github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= @@ -194,8 +157,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.5.0 h1:pLqT2kq1zpHW/1D18QMjMpdtX7cekxqtJJjg5ANyWw0= github.com/leodido/go-urn v1.5.0/go.mod h1:9BORnCDhdPBJNDEX+w1bJisa8yOKYi116VeO96s4ifE= -github.com/libgit2/git2go/v34 v34.0.0 h1:UKoUaKLmiCRbOCD3PtUi2hD6hESSXzME/9OUZrGcgu8= -github.com/libgit2/git2go/v34 v34.0.0/go.mod h1:blVco2jDAw6YTXkErMMqzHLcAjKkwF0aWIRHBqiJkZ0= github.com/magiconair/properties v1.18.12 h1:sT9zQpvTB3B4gzrX0tmZNTEaGyg8Zw55MFYRE32Mr9I= github.com/magiconair/properties v1.18.12/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -217,16 +178,12 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= -github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/panjf2000/ants/v2 v2.12.1 h1:BWvU2wHpyXWxhhNXsGB6JXLCNbshyLd1QxvoAmZnu10= github.com/panjf2000/ants/v2 v2.12.1/go.mod h1:tSQuaNQ6r6NRhPt+IZVUevvDyFMTs+eS4ztZc52uJTY= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pjbgf/sha1cd v0.7.0 h1:ZRNPKHj+gfkLBf0KJv/p1Hmz+7bqCT5o311YX0nq+DA= -github.com/pjbgf/sha1cd v0.7.0/go.mod h1:pKR5Li+qTCo+ebqITZfwPlJGoCFCvSO00qxjbNtTUFQ= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -243,13 +200,8 @@ github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52Z github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= -github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= -github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= @@ -269,9 +221,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -300,8 +250,6 @@ github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/viant/afs v1.30.1 h1:l+cIg7pVo5Cdia2GqaK+h3hwvj26QHg6VHmuatZ0CTM= github.com/viant/afs v1.30.1/go.mod h1:rScbFd9LJPGTM8HOI8Kjwee0AZ+MZMupAvFpPg+Qdj4= -github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= -github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -319,9 +267,7 @@ golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= @@ -337,7 +283,6 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= @@ -352,22 +297,16 @@ golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -395,18 +334,14 @@ google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fw google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/datatypes v1.2.7 h1:ww9GAhF1aGXZY3EB3cJPJ7//JiuQo7DlQA7NNlVaTdk= gorm.io/datatypes v1.2.7/go.mod h1:M2iO+6S3hhi4nAyYe444Pcb0dcIiOMJ7QHaUXxyiNZY= diff --git a/backend/plugins/gitextractor/README.md b/backend/plugins/gitextractor/README.md index 9d7067782..e1e026697 100644 --- a/backend/plugins/gitextractor/README.md +++ b/backend/plugins/gitextractor/README.md @@ -16,8 +16,9 @@ limitations under the License. --> Please see details in the [Apache DevLake website](https://devlake.apache.org/docs/Plugins/gitextractor) +The plugin reads repositories with the `git` command line, so `git` (2.31 or +newer) must be installed where the server runs. No libgit2 or cgo is required. -Build with local compiled v1.5.0 libgit2 ``` -CGO_LDFLAGS="-L/usr/local/lib -lgit2" CGO_CFLAGS="-I/usr/local/include" go build ./plugins/gitextractor/... 2>&1 | tail -5 +go build ./plugins/gitextractor/... ``` \ No newline at end of file diff --git a/backend/plugins/gitextractor/gitextractor.go b/backend/plugins/gitextractor/gitextractor.go index 1261c3834..da8f9b9ca 100644 --- a/backend/plugins/gitextractor/gitextractor.go +++ b/backend/plugins/gitextractor/gitextractor.go @@ -35,8 +35,6 @@ func main() { // pk := cmd.Flags().StringP("privateKey", "k", "", "private key file") // pkPass := cmd.Flags().StringP("privateKeyPassPhrase", "P", "", "passphrase for private key") proxy := cmd.Flags().StringP("proxy", "x", "", "proxy") - useGoGit := cmd.Flags().BoolP("useGoGit", "g", false, "use go-git instead of libgit2") - useGitCli := cmd.Flags().BoolP("useGitCli", "", false, "use the git command line collector") skipCommitStat := cmd.Flags().BoolP("skipCommitStat", "S", false, "") skipCommitFiles := cmd.Flags().BoolP("skipCommitFiles", "F", true, "") noShallowClone := cmd.Flags().BoolP("noShallowClone", "A", false, "") @@ -53,8 +51,6 @@ func main() { // "privateKey": * // "passphrase" "proxy": *proxy, - "useGoGit": *useGoGit, - "useGitCli": *useGitCli, "skipCommitStat": skipCommitStat, "skipCommitFiles": skipCommitFiles, "noShallowClone": noShallowClone, diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go index 746afee85..6b0aa1bc5 100644 --- a/backend/plugins/gitextractor/impl/impl.go +++ b/backend/plugins/gitextractor/impl/impl.go @@ -20,7 +20,6 @@ package impl import ( "fmt" "net/url" - "strings" "github.com/apache/devlake/core/dal" "github.com/apache/devlake/core/errors" @@ -116,12 +115,8 @@ func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[st } *optValue = &defValue } - loadBool(&op.UseGoGit, "UseGoGit", false) loadBool(&op.SkipCommitStat, "SKIP_COMMIT_STAT", false) loadBool(&op.SkipCommitFiles, "SKIP_COMMIT_FILES", true) - op.UseGitCli = op.UseGitCli || strings.EqualFold(cfg.GetString("GIT_EXTRACTOR_COLLECTOR"), "gitcli") - log.Info("UseGoGit: %v", *op.UseGoGit) - log.Info("UseGitCli: %v", op.UseGitCli) log.Info("SkipCommitStat: %v", *op.SkipCommitStat) log.Info("SkipCommitFiles: %v", *op.SkipCommitFiles) diff --git a/backend/plugins/gitextractor/models/diffline.go b/backend/plugins/gitextractor/models/diffline.go deleted file mode 100644 index 5375df294..000000000 --- a/backend/plugins/gitextractor/models/diffline.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package models - -import git "github.com/libgit2/git2go/v34" - -type DiffLines []git.DiffLine - -// Len return length of diffLines -func (diffLines DiffLines) Len() int { - return len(diffLines) -} - -// Less function -func (diffLines DiffLines) Less(i, j int) bool { - return diffLines[i].OldLineno > diffLines[j].OldLineno -} - -// Swap function -func (diffLines DiffLines) Swap(i, j int) { - temp := diffLines[i] - diffLines[i] = diffLines[j] - diffLines[j] = temp -} diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go index 3c88899f7..06f1a7acb 100644 --- a/backend/plugins/gitextractor/parser/clone_gitcli.go +++ b/backend/plugins/gitextractor/parser/clone_gitcli.go @@ -46,8 +46,6 @@ const maxExtraDeepenRounds = 10 // CloneRepoConfig is the configuration for the CloneRepo method // the subtask should run in Full Sync mode whenever the configuration is changed type CloneRepoConfig struct { - UseGoGit *bool - UseGitCli bool SkipCommitStat *bool SkipCommitFiles *bool NoShallowClone bool @@ -73,8 +71,6 @@ func NewGitcliCloner(ctx plugin.SubTaskContext, localDir string) (*GitcliCloner, SubTaskContext: ctx, Params: taskData.Options.GitExtractorApiParams, SubtaskConfig: CloneRepoConfig{ - UseGoGit: taskData.Options.UseGoGit, - UseGitCli: taskData.Options.UseGitCli, SkipCommitStat: taskData.Options.SkipCommitStat, SkipCommitFiles: taskData.Options.SkipCommitFiles, NoShallowClone: taskData.Options.NoShallowClone, @@ -97,7 +93,7 @@ func NewGitcliCloner(ctx plugin.SubTaskContext, localDir string) (*GitcliCloner, func (g *GitcliCloner) prepareSync() errors.Error { taskData := g.taskData // Disable automatic (detached) gc/maintenance after clone/fetch. Otherwise git may - // repack the repo in the background while libgit2 is already reading it, which + // repack the repo in the background while the collector is reading it, which // makes temporary pack files (.tmp-<pid>-pack-*.idx) vanish and the collection fail. g.globalConfigArgs = append(g.globalConfigArgs, "-c", "gc.auto=0", @@ -203,7 +199,7 @@ func (g *GitcliCloner) CloneRepo() errors.Error { if g.taskData.Options.NoShallowClone { // data source does not support shallow clone // 1. perform a full clone to accommodate - // 2. perform a local shallow clone to reduce the libgit2 memory usage + // 2. perform a local shallow clone so the collector reads a smaller repo if err := g.doubleClone(); err != nil { return err } diff --git a/backend/plugins/gitextractor/parser/clone_gitcli_test.go b/backend/plugins/gitextractor/parser/clone_gitcli_test.go index 790980f64..83636b97e 100644 --- a/backend/plugins/gitextractor/parser/clone_gitcli_test.go +++ b/backend/plugins/gitextractor/parser/clone_gitcli_test.go @@ -127,7 +127,7 @@ func TestIncrementalCloneKeepsCommitsBehindMergeBoundary(t *testing.T) { } // git fetch may start `git maintenance run --auto` / `git gc --auto` in the -// background. That repacks the repo while libgit2 is already reading it, so +// background. That repacks the repo while the collector is reading it, so // temporary pack files (.tmp-<pid>-pack-*.idx) vanish under its feet and the // collection fails with "could not find ... to open". func TestCloneDoesNotStartAutoMaintenance(t *testing.T) { diff --git a/backend/plugins/gitextractor/parser/collector_contract_test.go b/backend/plugins/gitextractor/parser/collector_contract_test.go index 52b97ece9..6e2bf8d07 100644 --- a/backend/plugins/gitextractor/parser/collector_contract_test.go +++ b/backend/plugins/gitextractor/parser/collector_contract_test.go @@ -19,6 +19,8 @@ package parser import ( "context" + "os" + "path/filepath" "sort" "testing" @@ -33,93 +35,20 @@ import ( "github.com/stretchr/testify/require" ) -// collectorCase describes one RepoCollector implementation under test. -// -// The assertions in this file describe the collector contract that a -// replacement implementation must satisfy. The deviations block records where -// an existing implementation is known to violate that contract; they are -// documented instead of being silently tolerated so that a replacement cannot -// inherit them. -type collectorCase struct { - name string - open func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector - - deviations collectorDeviations -} - -type collectorDeviations struct { - // storesOnlyLastComponent reports that only the component of the last file - // of each commit is stored. - storesOnlyLastComponent bool - // countTagsIgnoresContext reports that CountTags ignores a cancelled context. - countTagsIgnoresContext bool - // collectsShallowBoundary reports that the shallow boundary commit is - // collected with a full-tree diff instead of being skipped. - collectsShallowBoundary bool - // failsOnDeletedFiles reports that CollectDiffLine fails when the history - // contains a deleted file. - failsOnDeletedFiles bool - // skipsLineChanges reports that CollectDiffLine does not store - // CommitLineChange rows. - skipsLineChanges bool - // failsOnEmptyRepo reports that the Count* helpers fail on a repository - // without commits. - failsOnEmptyRepo bool - // omitsZeroLineFiles reports that changed files without line changes - // (binary files, empty files) are not stored. - omitsZeroLineFiles bool -} - -func collectorCases() []collectorCase { - return []collectorCase{ - { - name: "gitcli", - open: func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { - t.Helper() - collector, err := NewGitcliRepoCollector(localDir, repoId, store, unithelper.DummyLogger()) - require.Nil(t, err) - return collector - }, - }, - { - name: "libgit2", - open: func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { - t.Helper() - collector, err := NewLibgit2RepoCollector(localDir, repoId, store, unithelper.DummyLogger()) - require.Nil(t, err) - return collector - }, - deviations: collectorDeviations{ - storesOnlyLastComponent: true, - countTagsIgnoresContext: true, - collectsShallowBoundary: true, - }, - }, - { - name: "gogit", - open: func(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { - t.Helper() - collector, err := NewGogitRepoCollector(localDir, repoId, store, unithelper.DummyLogger()) - require.Nil(t, err) - return collector - }, - deviations: collectorDeviations{ - failsOnDeletedFiles: true, - skipsLineChanges: true, - failsOnEmptyRepo: true, - omitsZeroLineFiles: true, - }, - }, - } +// These tests describe the collector contract: what the git command line +// collector writes for a known repository. The fixtures are built with the git +// CLI itself, so the tests do not depend on any git library. + +func newTestCollector(t *testing.T, localDir, repoId string, store models.Store) RepoCollector { + t.Helper() + collector, err := NewGitcliRepoCollector(localDir, repoId, store, unithelper.DummyLogger()) + require.Nil(t, err) + return collector } -func forEachCollector(t *testing.T, fn func(t *testing.T, c collectorCase)) { +func forEachCollector(t *testing.T, fn func(t *testing.T)) { t.Helper() - for _, c := range collectorCases() { - t.Run(c.name, func(t *testing.T) { - fn(t, c) - }) - } + t.Run("gitcli", fn) } // recordStore is a models.Store that records everything the collectors write. @@ -275,9 +204,9 @@ func parentShas(parents []*code.CommitParent, commitSha string) []string { func TestCollectorCountsAndRefs(t *testing.T) { repo := buildCanonicalRepo(t) - forEachCollector(t, func(t *testing.T, c collectorCase) { + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, "test-repo", store) + collector := newTestCollector(t, repo.Dir, "test-repo", store) ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) commits, err := collector.CountCommits(context.Background()) @@ -329,9 +258,9 @@ func TestCollectorCommits(t *testing.T) { repo := buildCanonicalRepo(t) repoId := "test-repo" - forEachCollector(t, func(t *testing.T, c collectorCase) { + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, repoId, store) + collector := newTestCollector(t, repo.Dir, repoId, store) ctx := newCollectorContext(t, defaultOptions(repoId), nil) require.NoError(t, collector.CollectCommits(ctx)) @@ -392,20 +321,7 @@ func TestCollectorCommits(t *testing.T) { repo.C5: {"feature.txt": {1, 0}}, } for sha, want := range expectedFiles { - if !c.deviations.omitsZeroLineFiles { - assert.Equal(t, want, files[sha], sha) - continue - } - for path, stat := range want { - if stat != [2]int{0, 0} { - assert.Equal(t, stat, files[sha][path], "%s %s", sha, path) - } - } - for path, stat := range files[sha] { - if stat != [2]int{0, 0} { - assert.Contains(t, want, path, "%s %s", sha, path) - } - } + assert.Equal(t, want, files[sha], sha) } assert.Equal(t, 4, commits[repo.C1].Additions) @@ -421,11 +337,9 @@ func TestCollectorCommits(t *testing.T) { for _, file := range store.commitFiles { assert.Equal(t, genCommitFileId(file.CommitSha, file.FilePath), file.Id) } - if !c.deviations.storesOnlyLastComponent { - components := componentsById(store.components) - for _, file := range store.commitFiles { - assert.Equal(t, "Default", components[file.Id], file.FilePath) - } + components := componentsById(store.components) + for _, file := range store.commitFiles { + assert.Equal(t, "Default", components[file.Id], file.FilePath) } }) } @@ -433,9 +347,9 @@ func TestCollectorCommits(t *testing.T) { func TestCollectorUsesConfiguredComponents(t *testing.T) { repo := buildLinearRepo(t) - forEachCollector(t, func(t *testing.T, c collectorCase) { + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, "test-repo", store) + collector := newTestCollector(t, repo.Dir, "test-repo", store) ctx := newCollectorContext(t, defaultOptions("test-repo"), []code.Component{ {RepoId: "test-repo", Name: "docs", PathRegex: `^c\.txt$`}, }) @@ -446,18 +360,16 @@ func TestCollectorUsesConfiguredComponents(t *testing.T) { files := filesByCommit(store.commitFiles)[repo.C4] require.Contains(t, files, "c.txt") assert.Equal(t, "docs", components[genCommitFileId(repo.C4, "c.txt")]) - if !c.deviations.storesOnlyLastComponent { - assert.Equal(t, "Default", components[genCommitFileId(repo.C4, "b.txt")]) - } + assert.Equal(t, "Default", components[genCommitFileId(repo.C4, "b.txt")]) }) } func TestCollectorSkipCommitStat(t *testing.T) { repo := buildCanonicalRepo(t) - forEachCollector(t, func(t *testing.T, c collectorCase) { + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, "test-repo", store) + collector := newTestCollector(t, repo.Dir, "test-repo", store) opts := defaultOptions("test-repo") opts.SkipCommitStat = boolPtr(true) opts.SkipCommitFiles = boolPtr(true) @@ -478,9 +390,9 @@ func TestCollectorSkipCommitStat(t *testing.T) { func TestCollectorSkipCommitFiles(t *testing.T) { repo := buildCanonicalRepo(t) - forEachCollector(t, func(t *testing.T, c collectorCase) { + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, "test-repo", store) + collector := newTestCollector(t, repo.Dir, "test-repo", store) opts := defaultOptions("test-repo") opts.SkipCommitFiles = boolPtr(true) ctx := newCollectorContext(t, opts, nil) @@ -498,9 +410,9 @@ func TestCollectorSkipCommitFiles(t *testing.T) { func TestCollectorExcludeFileExtensions(t *testing.T) { repo := buildCanonicalRepo(t) - forEachCollector(t, func(t *testing.T, c collectorCase) { + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, "test-repo", store) + collector := newTestCollector(t, repo.Dir, "test-repo", store) opts := defaultOptions("test-repo") opts.ExcludeFileExtensions = []string{".DAT"} ctx := newCollectorContext(t, opts, nil) @@ -529,12 +441,9 @@ func TestCollectorSkipsShallowBoundaryCommit(t *testing.T) { dir := f.shallowBareClone() - forEachCollector(t, func(t *testing.T, c collectorCase) { - if c.deviations.collectsShallowBoundary { - t.Skip("known deviation: the shallow boundary is collected with a full-tree diff") - } + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, dir, "test-repo", store) + collector := newTestCollector(t, dir, "test-repo", store) ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) // The tip is a shallow boundary: its parent is not in the object @@ -546,15 +455,42 @@ func TestCollectorSkipsShallowBoundaryCommit(t *testing.T) { }) } +func TestCollectorSkipsShallowBoundaryEvenWhenItsParentExists(t *testing.T) { + f := newGitFixture(t) + f.writeFile("a.txt", "l1\n") + c1 := f.commit(testTimeC1, "c1") + f.writeFile("a.txt", "l1\nl2\n") + c2 := f.commit(testTimeC2, "c2") + // keep the parent reachable through another branch + f.git("branch", "old", c1) + + dir := f.bareClone() + // Mark the tip as a shallow boundary although its parent object is still + // in the object database. git then renders the tip as a root commit, so + // the collector must skip it based on the shallow file, not on the parent + // being missing. + require.NoError(t, os.WriteFile(filepath.Join(dir, "shallow"), []byte(c2+"\n"), 0o644)) + + forEachCollector(t, func(t *testing.T) { + store := &recordStore{} + collector := newTestCollector(t, dir, "test-repo", store) + ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) + + require.NoError(t, collector.CollectCommits(ctx)) + + commits := commitsBySha(store.commits) + assert.NotContains(t, commits, c2, "the shallow boundary must not be collected") + require.Contains(t, commits, c1) + assert.Equal(t, 1, commits[c1].Additions) + }) +} + func TestCollectorEmptyRepo(t *testing.T) { dir := newGitFixture(t).bareClone() - forEachCollector(t, func(t *testing.T, c collectorCase) { - if c.deviations.failsOnEmptyRepo { - t.Skip("known deviation: the Count* helpers need at least one commit") - } + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, dir, "test-repo", store) + collector := newTestCollector(t, dir, "test-repo", store) ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) commits, err := collector.CountCommits(context.Background()) @@ -578,39 +514,24 @@ func TestCollectorEmptyRepo(t *testing.T) { func TestCollectorHonoursCancelledContext(t *testing.T) { repo := buildCanonicalRepo(t) - forEachCollector(t, func(t *testing.T, c collectorCase) { + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, "test-repo", store) + collector := newTestCollector(t, repo.Dir, "test-repo", store) ctx, cancel := context.WithCancel(context.Background()) cancel() _, err := collector.CountCommits(ctx) assert.ErrorIs(t, err, context.Canceled) - if !c.deviations.countTagsIgnoresContext { - _, err = collector.CountTags(ctx) - assert.ErrorIs(t, err, context.Canceled) - } + _, err = collector.CountTags(ctx) + assert.ErrorIs(t, err, context.Canceled) _, err = collector.CountBranches(ctx) assert.ErrorIs(t, err, context.Canceled) }) } func TestCollectorRejectsNonRepo(t *testing.T) { - dir := t.TempDir() - for _, c := range collectorCases() { - t.Run(c.name, func(t *testing.T) { - var err error - switch c.name { - case "gitcli": - _, err = NewGitcliRepoCollector(dir, "test-repo", nil, unithelper.DummyLogger()) - case "libgit2": - _, err = NewLibgit2RepoCollector(dir, "test-repo", nil, unithelper.DummyLogger()) - case "gogit": - _, err = NewGogitRepoCollector(dir, "test-repo", nil, unithelper.DummyLogger()) - } - assert.Error(t, err, "%s should reject a non-repository", c.name) - }) - } + _, err := NewGitcliRepoCollector(t.TempDir(), "test-repo", nil, unithelper.DummyLogger()) + assert.Error(t, err, "the collector should reject a non-repository") } func TestCollectorCountsWhenHeadIsNotDefault(t *testing.T) { @@ -623,9 +544,9 @@ func TestCollectorCountsWhenHeadIsNotDefault(t *testing.T) { require.NotEqual(t, c1, c2) dir := f.bareClone() - forEachCollector(t, func(t *testing.T, c collectorCase) { + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, dir, "test-repo", store) + collector := newTestCollector(t, dir, "test-repo", store) // HEAD points at feature, which is not counted but is the default ref. branches, err := collector.CountBranches(context.Background()) @@ -648,12 +569,9 @@ func TestCollectorDiffLineSnapshot(t *testing.T) { repo := buildLinearRepo(t) expected := blameAtHEAD(t, repo.Dir) - forEachCollector(t, func(t *testing.T, c collectorCase) { - if c.deviations.failsOnDeletedFiles { - t.Skip("known deviation: CollectDiffLine fails on a deleted file") - } + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, "test-repo", store) + collector := newTestCollector(t, repo.Dir, "test-repo", store) ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) require.NoError(t, collector.CollectDiffLine(ctx)) @@ -672,21 +590,16 @@ func TestCollectorDiffLineSnapshot(t *testing.T) { assert.Equal(t, lines, got[file], file) } - if c.deviations.skipsLineChanges { - assert.Empty(t, store.lineChanges) - } + assert.NotEmpty(t, store.lineChanges) }) } func TestCollectorCollectAllAndClose(t *testing.T) { repo := buildCanonicalRepo(t) - forEachCollector(t, func(t *testing.T, c collectorCase) { - if c.deviations.failsOnDeletedFiles { - t.Skip("known deviation: CollectDiffLine fails on a deleted file") - } + forEachCollector(t, func(t *testing.T) { store := &recordStore{} - collector := c.open(t, repo.Dir, "test-repo", store) + collector := newTestCollector(t, repo.Dir, "test-repo", store) ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) cleanedUp := false @@ -698,11 +611,7 @@ func TestCollectorCollectAllAndClose(t *testing.T) { assert.Len(t, store.refs, 4) assert.NotEmpty(t, store.commitFiles) assert.NotEmpty(t, store.snapshots) - if c.deviations.skipsLineChanges { - assert.Empty(t, store.lineChanges) - } else { - assert.NotEmpty(t, store.lineChanges) - } + assert.NotEmpty(t, store.lineChanges) require.NoError(t, collector.Close(context.Background())) assert.True(t, store.closed) @@ -710,12 +619,11 @@ func TestCollectorCollectAllAndClose(t *testing.T) { }) } -func TestLibgit2CollectorCommitLineChanges(t *testing.T) { +func TestCollectorCommitLineChanges(t *testing.T) { repo := buildLinearRepo(t) store := &recordStore{} - collector, err := NewLibgit2RepoCollector(repo.Dir, "test-repo", store, unithelper.DummyLogger()) - require.Nil(t, err) + collector := newTestCollector(t, repo.Dir, "test-repo", store) ctx := newCollectorContext(t, defaultOptions("test-repo"), nil) require.NoError(t, collector.CollectDiffLine(ctx)) diff --git a/backend/plugins/gitextractor/parser/commitparse_test.go b/backend/plugins/gitextractor/parser/commitparse_test.go index 473fc070c..fca02f31f 100644 --- a/backend/plugins/gitextractor/parser/commitparse_test.go +++ b/backend/plugins/gitextractor/parser/commitparse_test.go @@ -34,7 +34,7 @@ parent 2222222222222222222222222222222222222222 author Jane Doe <[email protected]> 1704067200 +0800 committer DevLake Test <[email protected]> 1704070800 -0530 gpgsig -----BEGIN PGP SIGNATURE----- - + fakesignature-line fakesignature -----END PGP SIGNATURE----- diff --git a/backend/plugins/gitextractor/parser/numstatparse.go b/backend/plugins/gitextractor/parser/numstatparse.go index 1d96dd6ae..f7d3fc21e 100644 --- a/backend/plugins/gitextractor/parser/numstatparse.go +++ b/backend/plugins/gitextractor/parser/numstatparse.go @@ -117,7 +117,7 @@ func parseNumstatEntry(entry string) (commitFileStat, error) { } // parseNumstatCount parses a numstat count. Binary files are reported as "-" -// and count as zero changed lines, just like libgit2 reports them. +// and count as zero changed lines. func parseNumstatCount(value string) (int, error) { if value == "-" { return 0, nil diff --git a/backend/plugins/gitextractor/parser/parse_test.go b/backend/plugins/gitextractor/parser/parse_test.go index 23783c329..988a4b87d 100644 --- a/backend/plugins/gitextractor/parser/parse_test.go +++ b/backend/plugins/gitextractor/parser/parse_test.go @@ -157,7 +157,7 @@ func TestParsePatchStream(t *testing.T) { // binary file: both paths are known, but there are no line changes assert.Equal(t, patchFile{OldPath: "bin.dat", NewPath: "bin.dat"}, c2.Files[1]) - // deleted file: libgit2 reports the path on both sides + // deleted file: the path is reported on both sides assert.Equal(t, patchFile{ OldPath: "empty.txt", NewPath: "empty.txt", @@ -211,15 +211,19 @@ func TestParseGitVersion(t *testing.T) { for _, tc := range []struct { output string want int + ok bool }{ - {"git version 2.43.0\n", 2*10000 + 43*100}, - {"git version 2.31.0", 2*10000 + 31*100}, - {"git version 2.39.2.windows.1", 2*10000 + 39*100 + 2}, - {"git version 2.56.0", 2*10000 + 56*100}, + {"git version 2.43.0\n", 2*10000 + 43*100, true}, + {"git version 2.31.0", 2*10000 + 31*100, true}, + {"git version 2.39.2.windows.1", 2*10000 + 39*100 + 2, true}, + {"git version 2.56.0", 2*10000 + 56*100, true}, + {"git version 2.30.2", 2*10000 + 30*100 + 2, false}, + {"git version 1.9.0", 1*10000 + 9*100, false}, } { got, err := parseGitVersion(tc.output) if assert.NoError(t, err, tc.output) { assert.Equal(t, tc.want, got, tc.output) + assert.Equal(t, tc.ok, got >= minGitVersion, tc.output) } } @@ -241,3 +245,9 @@ func TestUnquoteGitPath(t *testing.T) { assert.Equal(t, tc.want, unquoteGitPath(tc.in), tc.in) } } + +func TestResolveRepoPath(t *testing.T) { + assert.Equal(t, "/repo/.git/shallow", resolveRepoPath("/repo", ".git/shallow")) + assert.Equal(t, "/repo/shallow", resolveRepoPath("/repo", "shallow")) + assert.Equal(t, "/abs/shallow", resolveRepoPath("/repo", "/abs/shallow")) +} diff --git a/backend/plugins/gitextractor/parser/patchparse.go b/backend/plugins/gitextractor/parser/patchparse.go index e4d3c8dc8..3aa27a70f 100644 --- a/backend/plugins/gitextractor/parser/patchparse.go +++ b/backend/plugins/gitextractor/parser/patchparse.go @@ -203,8 +203,8 @@ func parseHunkRange(value string) (int, error) { func (p *patchParser) endFile() { if p.current != nil { - // libgit2 reports the same path on both sides for additions and - // deletions, so mirror that here. + // Report the same path on both sides for additions and deletions, so the + // commit file and line change rows always name the file. if p.current.OldPath == "" { p.current.OldPath = p.current.NewPath } diff --git a/backend/plugins/gitextractor/parser/repo_gitcli.go b/backend/plugins/gitextractor/parser/repo_gitcli.go index 4cf1fc0d1..f13256daa 100644 --- a/backend/plugins/gitextractor/parser/repo_gitcli.go +++ b/backend/plugins/gitextractor/parser/repo_gitcli.go @@ -20,9 +20,12 @@ package parser import ( "bufio" "context" + "crypto/sha256" + "encoding/hex" "fmt" "io" "os" + "path/filepath" "regexp" "sort" "strconv" @@ -39,7 +42,7 @@ import ( // minGitVersion is the first release with --diff-merges=first-parent, which is // required to diff merge commits against their first parent only. -const minGitVersion = 2*10000 + 31 +const minGitVersion = 2*10000 + 31*100 // commitBatchSize bounds how many commits are fetched and diffed at a time so // that a repository with a long history does not need to be held in memory. @@ -47,8 +50,9 @@ const commitBatchSize = 1024 var _ RepoCollector = (*GitcliRepoCollector)(nil) -// GitcliRepoCollector collects git data with the git command line, so the -// server no longer needs libgit2 or a pure Go git implementation. +// GitcliRepoCollector collects git data with the git command line. It replaces +// the previous libgit2 and go-git implementations, so the server no longer +// needs a git library or cgo. type GitcliRepoCollector struct { id string logger log.Logger @@ -494,7 +498,7 @@ type snapshotChanges struct { added []patchLine } -// applySnapshotChanges mirrors the line bookkeeping of the libgit2 collector: +// applySnapshotChanges applies the changes of one file to the line snapshot: // deletions remove the old line numbers and additions insert the commit sha at // the new line numbers. func applySnapshotChanges(snapshot map[string]*models.FileBlame, filePath, commitSha string, changes *snapshotChanges) { @@ -588,7 +592,7 @@ func (r *GitcliRepoCollector) shallowCommits(ctx context.Context) (map[string]st if err != nil { return shallow, err } - content, err := os.ReadFile(strings.TrimSpace(string(out))) + content, err := os.ReadFile(resolveRepoPath(r.git.dir, strings.TrimSpace(string(out)))) if os.IsNotExist(err) { return shallow, nil } @@ -603,6 +607,16 @@ func (r *GitcliRepoCollector) shallowCommits(ctx context.Context) (map[string]st return shallow, nil } +// resolveRepoPath turns a git path that is relative to the repository into an +// absolute one. `git rev-parse --git-path` returns paths relative to the +// repository root, but the collector's working directory is not the repository. +func resolveRepoPath(repoDir, gitPath string) string { + if filepath.IsAbs(gitPath) { + return gitPath + } + return filepath.Join(repoDir, gitPath) +} + type tagRef struct { refname string sha string @@ -684,6 +698,15 @@ func sortedComponentNames(componentMap map[string]*regexp.Regexp) []string { return names } +// genCommitFileId builds the commit file primary key. With some long paths the +// varchar(255) columns were not enough for both the id and the file path, so +// the path is compressed with sha256. +func genCommitFileId(commitSha, filePath string) string { + shaFilePath := sha256.New() + shaFilePath.Write([]byte(filePath)) + return commitSha + ":" + hex.EncodeToString(shaFilePath.Sum(nil)) +} + func normalizeExtensions(extensions []string) map[string]struct{} { excluded := make(map[string]struct{}, len(extensions)) for _, extension := range extensions { diff --git a/backend/plugins/gitextractor/parser/repo_gogit.go b/backend/plugins/gitextractor/parser/repo_gogit.go deleted file mode 100644 index fe254e859..000000000 --- a/backend/plugins/gitextractor/parser/repo_gogit.go +++ /dev/null @@ -1,626 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package parser - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "regexp" - "strings" - - "github.com/apache/devlake/core/dal" - "github.com/apache/devlake/core/errors" - "github.com/apache/devlake/core/log" - "github.com/apache/devlake/core/models/domainlayer" - "github.com/apache/devlake/core/models/domainlayer/code" - "github.com/apache/devlake/core/plugin" - "github.com/apache/devlake/plugins/gitextractor/models" - gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/storer" -) - -type GogitRepoCollector struct { - id string - logger log.Logger - store models.Store - repo *gogit.Repository - cleanUp func() -} - -func NewGogitRepoCollector(localDir string, repoId string, store models.Store, logger log.Logger) (*GogitRepoCollector, errors.Error) { - repo, err := gogit.PlainOpen(localDir) - if err != nil { - return nil, errors.Convert(err) - } - return &GogitRepoCollector{ - id: repoId, - logger: logger, - store: store, - repo: repo, - }, nil -} - -func (r *GogitRepoCollector) SetCleanUp(f func()) error { - if f != nil { - r.cleanUp = f - } - return nil -} - -func (r *GogitRepoCollector) Close(ctx context.Context) error { - if err := r.store.Close(); err != nil { - return err - } - if r.cleanUp != nil { - r.cleanUp() - } - return nil -} - -// CollectAll The main parser subtask -func (r *GogitRepoCollector) CollectAll(subtaskCtx plugin.SubTaskContext) error { - subtaskCtx.SetProgress(0, -1) - err := r.CollectTags(subtaskCtx) - if err != nil { - return err - } - err = r.CollectBranches(subtaskCtx) - if err != nil { - return err - } - err = r.CollectCommits(subtaskCtx) - if err != nil { - return err - } - return r.CollectDiffLine(subtaskCtx) -} - -// CountTags Count git tags subtask -func (r *GogitRepoCollector) CountTags(ctx context.Context) (int, error) { - iter, err := r.repo.Tags() - if err != nil { - return 0, err - } - var tagsCount int - if err := iter.ForEach(func(reference *plumbing.Reference) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - tagsCount += 1 - return nil - }); err != nil { - return 0, err - } - return tagsCount, nil -} - -// CountBranches count the number of branches in a git repo -func (r *GogitRepoCollector) CountBranches(ctx context.Context) (int, error) { - refIter, err := r.repo.Storer.IterReferences() - if err != nil { - return 0, err - } - branchIter := storer.NewReferenceFilteredIter( - func(r *plumbing.Reference) bool { - return r.Name().IsBranch() || r.Name().IsRemote() - }, refIter) - var branchesCount int - - headRef, err := r.repo.Head() - if err != nil { - return 0, err - } - if err := branchIter.ForEach(func(reference *plumbing.Reference) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - if reference.Name() != headRef.Name() { - branchesCount += 1 - } - return nil - }); err != nil { - return 0, err - } - return branchesCount, nil -} - -// CountCommits count the number of commits in a git repo -func (r *GogitRepoCollector) CountCommits(ctx context.Context) (int, error) { - iter, err := r.repo.CommitObjects() - if err != nil { - return 0, err - } - var count int - if err := iter.ForEach(func(commit *object.Commit) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - count += 1 - return nil - }); err != nil { - return 0, err - } - return count, nil -} - -// CollectTags Collect Tags data -func (r *GogitRepoCollector) CollectTags(subtaskCtx plugin.SubTaskContext) error { - tagIter, err := r.repo.Tags() - if err != nil { - return err - } - if err := tagIter.ForEach(func(ref *plumbing.Reference) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - tagCommit := ref.Hash().String() - _, err := r.repo.CommitObject(ref.Hash()) - if err != nil && errors.Is(err, plumbing.ErrObjectNotFound) { - h, err := r.repo.ResolveRevision(plumbing.Revision(ref.Name())) - if err != nil { - return err - } - tagCommit = h.String() - } - name := ref.Name().String() - if tagCommit != "" { - codeRef := &code.Ref{ - DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, name)}, - RepoId: r.id, - Name: name, - CommitSha: tagCommit, - RefType: TAG, - } - err = r.store.Refs(codeRef) - if err != nil { - return err - } - subtaskCtx.IncProgress(1) - } - return nil - }); err != nil { - return err - } - return nil -} - -// CollectBranches Collect branch data -func (r *GogitRepoCollector) CollectBranches(subtaskCtx plugin.SubTaskContext) error { - refIter, err := r.repo.Storer.IterReferences() - if err != nil { - return err - } - branchIter := storer.NewReferenceFilteredIter( - func(r *plumbing.Reference) bool { - return r.Name().IsBranch() || r.Name().IsRemote() - }, refIter) - headRef, err := r.repo.Head() - if err != nil { - return err - } - if err := branchIter.ForEach(func(ref *plumbing.Reference) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - name := ref.Name().Short() - sha := ref.Hash().String() - _, err := r.repo.CommitObject(ref.Hash()) - if err != nil && errors.Is(err, plumbing.ErrObjectNotFound) { - // handle commit sha like "0000000000000000000000000000000000000000" - h, err := r.repo.ResolveRevision(plumbing.Revision(ref.Name())) - if err != nil { - return err - } - sha = h.String() - } - codeRef := &code.Ref{ - DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, name)}, - RepoId: r.id, - Name: name, - CommitSha: sha, - RefType: BRANCH, - IsDefault: ref.Name() == headRef.Name(), - } - if err := r.store.Refs(codeRef); err != nil { - return err - } - subtaskCtx.IncProgress(1) - return nil - }); err != nil { - return err - } - return nil -} - -func (r *GogitRepoCollector) getComponentMap(subtaskCtx plugin.SubTaskContext) (map[string]*regexp.Regexp, error) { - db := subtaskCtx.GetDal() - components := make([]code.Component, 0) - err := db.All(&components, dal.From(components), dal.Where("repo_id= ?", r.id)) - if err != nil { - return nil, err - } - componentMap := make(map[string]*regexp.Regexp) - for _, component := range components { - componentMap[component.Name] = regexp.MustCompile(component.PathRegex) - } - return componentMap, nil -} - -// CollectCommits Collect data from each commit, we can also get the diff line -func (r *GogitRepoCollector) CollectCommits(subtaskCtx plugin.SubTaskContext) (err error) { - taskOpts := subtaskCtx.GetData().(*GitExtractorTaskData).Options - // check it first - componentMap, err := r.getComponentMap(subtaskCtx) - if err != nil { - return err - } - - repo := r.repo - store := r.store - - commitsObjectsIter, err := repo.CommitObjects() - if err != nil { - return err - } - - if err := commitsObjectsIter.ForEach(func(commit *object.Commit) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - commitSha := commit.Hash.String() - - if commit.NumParents() != 0 { - _, err := commit.Parents().Next() - if err != nil { - if err == plumbing.ErrObjectNotFound { - // Skip calculating commit statistics when there are parent commits, but the first one cannot be fetched from the ODB. - // This usually happens during a shallow clone for incremental collection. Otherwise, we might end up overwriting - // the correct addition/deletion data in the database with an absurdly large addition number. - r.logger.Info("skip commit %s because it has no parent commit", commitSha) - return nil - } - return err - } - } - codeCommit := &code.Commit{ - Sha: commitSha, - Message: commit.Message, - AuthorName: commit.Author.Name, - AuthorEmail: commit.Author.Email, - AuthorId: commit.Author.Email, - AuthoredDate: commit.Author.When, - CommitterName: commit.Committer.Name, - CommitterEmail: commit.Committer.Email, - CommitterId: commit.Committer.Email, - CommittedDate: commit.Committer.When, - } - if err = r.storeParentCommits(commitSha, commit); err != nil { - return err - } - - if !*taskOpts.SkipCommitStat { - stats, err := commit.StatsContext(subtaskCtx.GetContext()) - if err != nil { - return err - } else { - excluded := map[string]struct{}{} - for _, ext := range taskOpts.ExcludeFileExtensions { - e := strings.ToLower(strings.TrimSpace(ext)) - if e == "" { - continue - } - excluded[e] = struct{}{} - } - for _, stat := range stats { - nameLower := strings.ToLower(stat.Name) - skip := false - for ext := range excluded { - if strings.HasSuffix(nameLower, ext) { - skip = true - break - } - } - if skip { - continue - } - codeCommit.Additions += stat.Addition - // In some repos, deletion may be zero, which is different from git log --stat. - // It seems go-git doesn't get the correct changes. - // I have run object.DiffTreeWithOptions manually with different diff algorithms, - // but get the same result with StatsContext. - // I cannot reproduce it with another repo. - // A similar issue: https://github.com/go-git/go-git/issues/367 - codeCommit.Deletions += stat.Deletion - } - } - } - - err = store.Commits(codeCommit) - if err != nil { - return err - } - - codeRepoCommit := &code.RepoCommit{ - RepoId: r.id, - CommitSha: commitSha, - } - err = store.RepoCommits(codeRepoCommit) - if err != nil { - return err - } - if !*taskOpts.SkipCommitFiles { - if err := r.storeDiffCommitFilesComparedToParent(subtaskCtx, componentMap, commit, taskOpts.ExcludeFileExtensions); err != nil { - return err - } - } - subtaskCtx.IncProgress(1) - return nil - }); err != nil { - return err - } - return -} - -func (r *GogitRepoCollector) storeParentCommits(commitSha string, commit *object.Commit) error { - if commit == nil { - return nil - } - var commitParents []*code.CommitParent - for i := 0; i < commit.NumParents(); i++ { - parent, err := commit.Parent(i) - if err != nil { - // parent commit might not exist when repo is shallow cloned (tradeoff of supporting timeAfter parameter) - if err.Error() == "object not found" { - continue - } - return err - } - if parent != nil { - if parentCommitSha := parent.Hash.String(); parentCommitSha != "" { - commitParents = append(commitParents, &code.CommitParent{ - CommitSha: commitSha, - ParentCommitSha: parentCommitSha, - }) - } - } - } - return r.store.CommitParents(commitParents) -} - -func (r *GogitRepoCollector) getCurrentAndParentTree(ctx context.Context, commit *object.Commit) (*object.Tree, *object.Tree, error) { - if _, err := commit.Stats(); err != nil { - return nil, nil, err - } - commitTree, err := commit.Tree() - if err != nil { - return nil, nil, err - } - var firstParentTree *object.Tree - if commit.NumParents() > 0 { - firstParent, err := commit.Parents().Next() - if err != nil { - return nil, nil, err - } - firstParentTree, err = firstParent.Tree() - if err != nil { - return nil, nil, err - } - } - return commitTree, firstParentTree, nil -} - -func (r *GogitRepoCollector) storeDiffCommitFilesComparedToParent(subtaskCtx plugin.SubTaskContext, componentMap map[string]*regexp.Regexp, commit *object.Commit, excludeExts []string) (err error) { - commitTree, firstParentTree, err := r.getCurrentAndParentTree(subtaskCtx.GetContext(), commit) - if err != nil { - return err - } - // no parent, doesn't need to patch - patch, err := firstParentTree.PatchContext(subtaskCtx.GetContext(), commitTree) - if err != nil { - return err - } - // normalize exclusions - excluded := map[string]struct{}{} - for _, ext := range excludeExts { - e := strings.ToLower(strings.TrimSpace(ext)) - if e == "" { - continue - } - excluded[e] = struct{}{} - } - for _, p := range patch.Stats() { - commitFile := &code.CommitFile{ - CommitSha: commit.Hash.String(), - } - fileName := p.Name - commitFile.FilePath = fileName - if len(excluded) > 0 { - lower := strings.ToLower(fileName) - skip := false - for ext := range excluded { - if strings.HasSuffix(lower, ext) { - skip = true - break - } - } - if skip { - continue - } - } - commitFile.Id = genCommitFileId(commitFile.CommitSha, fileName) - commitFile.Deletions = p.Deletion - commitFile.Additions = p.Addition - if err := r.storeCommitFileComponents(subtaskCtx, componentMap, commitFile.Id, commitFile.FilePath); err != nil { - return err - } - err = r.store.CommitFiles(commitFile) - if err != nil { - r.logger.Error(err, "CommitFiles error") - return nil - } - } - return nil -} - -// With some long path,the varchar(255) was not enough both ID and file_path -// So we use the hash to compress the path in ID and add length of file_path. -// Use commitSha and the sha256 of FilePath to create id -func genCommitFileId(commitSha, filePath string) string { - shaFilePath := sha256.New() - shaFilePath.Write([]byte(filePath)) - return commitSha + ":" + hex.EncodeToString(shaFilePath.Sum(nil)) -} - -func (r *GogitRepoCollector) storeCommitFileComponents(subtaskCtx plugin.SubTaskContext, componentMap map[string]*regexp.Regexp, commitFileId string, commitFilePath string) error { - if commitFileId == "" || commitFilePath == "" { - return errors.Default.New("commit id r commit file path is empty") - } - commitFileComponent := &code.CommitFileComponent{ - CommitFileId: commitFileId, - ComponentName: "Default", - } - for component, reg := range componentMap { - if reg.MatchString(commitFilePath) { - commitFileComponent.ComponentName = component - break - } - } - return r.store.CommitFileComponents(commitFileComponent) -} - -// storeRepoSnapshot depends on commit list's order. -func (r *GogitRepoCollector) storeRepoSnapshot(subtaskCtx plugin.SubTaskContext, commitList []*object.Commit) error { - ctx := subtaskCtx.GetContext() - snapshot := make(map[string][]string) // {"filePathAndName": ["line1 commit sha", "line2 commit sha"]} - for _, commit := range commitList { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - commitTree, firstParentTree, err := r.getCurrentAndParentTree(ctx, commit) - if err != nil { - return err - } - patch, err := firstParentTree.PatchContext(subtaskCtx.GetContext(), commitTree) - if err != nil { - return err - } - for _, p := range patch.Stats() { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - fileName := p.Name - if _, ok := snapshot[fileName]; !ok { - snapshot[fileName] = []string{} - } - blameResults, err := gogit.Blame(commit, fileName) - if err != nil { - return err - } - var newBlames []string - for _, blameResult := range blameResults.Lines { - newBlames = append(newBlames, blameResult.Hash.String()) - } - snapshot[fileName] = newBlames - } - } - // store snapshots - for fileName, lineBlames := range snapshot { - for idx, lineBlameHash := range lineBlames { - lineNo := idx + 1 - repoSnapshot := &code.RepoSnapshot{ - RepoId: r.id, - CommitSha: lineBlameHash, - FilePath: fileName, - LineNo: lineNo, - } - if err := r.store.RepoSnapshot(repoSnapshot); err != nil { - r.logger.Error(err, "store RepoSnapshot error") - return err - } - } - } - return nil -} - -func (r *GogitRepoCollector) GetCommitList(subtaskCtx plugin.SubTaskContext) ([]*object.Commit, error) { - var commitList []*object.Commit - // get current head commit sha, default is master branch - // check branch, if not master, checkout to branch's head - commitOid, err := r.repo.Head() - if err != nil { - return nil, err - } - // get head commit object and add into commitList - commit, err := r.repo.CommitObject(commitOid.Hash()) - if err != nil { - return nil, err - } - commitList = append(commitList, commit) - // if current head has parents, get parent commit sha - for commit != nil && commit.NumParents() > 0 { - parentCommit, err := commit.Parent(0) - if err != nil { - return nil, err - } - commit, err = r.repo.CommitObject(parentCommit.Hash) - if err != nil { - return nil, err - } - commitList = append(commitList, commit) - } - // reverse commitList - // use slices.Reverse(commitList) in higher golang version. - for i, j := 0, len(commitList)-1; i < j; i, j = i+1, j-1 { - commitList[i], commitList[j] = commitList[j], commitList[i] - } - return commitList, nil -} - -func (r *GogitRepoCollector) CollectDiffLine(subtaskCtx plugin.SubTaskContext) error { - commitList, err := r.GetCommitList(subtaskCtx) - if err != nil { - return err - } - if err := r.storeRepoSnapshot(subtaskCtx, commitList); err != nil { - return err - } - // fixme: collecting CommitLineChange is not implemented. - // There is no way to get such information with go-git, and table commit_line_change is not used by any dashboards - // So we just ignore it. - return nil -} diff --git a/backend/plugins/gitextractor/parser/repo_libgit2.go b/backend/plugins/gitextractor/parser/repo_libgit2.go deleted file mode 100644 index ad9b515b4..000000000 --- a/backend/plugins/gitextractor/parser/repo_libgit2.go +++ /dev/null @@ -1,698 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package parser - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "regexp" - "sort" - "strconv" - "strings" - - "github.com/apache/devlake/core/dal" - "github.com/apache/devlake/core/errors" - "github.com/apache/devlake/core/log" - "github.com/apache/devlake/core/models/domainlayer" - "github.com/apache/devlake/core/models/domainlayer/code" - "github.com/apache/devlake/core/plugin" - "github.com/apache/devlake/plugins/gitextractor/models" - - git "github.com/libgit2/git2go/v34" -) - -var _ RepoCollector = (*Libgit2RepoCollector)(nil) - -var TypeNotMatchError = "the requested type does not match the type in the ODB" - -type Libgit2RepoCollector struct { - id string - logger log.Logger - - store models.Store - repo *git.Repository - cleanup func() -} - -func NewLibgit2RepoCollector(localDir string, repoId string, store models.Store, logger log.Logger) (*Libgit2RepoCollector, errors.Error) { - repo, err := git.OpenRepository(localDir) - if err != nil { - return nil, errors.Convert(err) - } - return &Libgit2RepoCollector{ - id: repoId, - logger: logger, - store: store, - repo: repo, - }, nil -} - -func (r *Libgit2RepoCollector) SetCleanUp(f func()) error { - if f != nil { - r.cleanup = f - } - return nil -} - -// CollectAll The main parser subtask -func (r *Libgit2RepoCollector) CollectAll(subtaskCtx plugin.SubTaskContext) error { - subtaskCtx.SetProgress(0, -1) - err := r.CollectTags(subtaskCtx) - if err != nil { - return err - } - err = r.CollectBranches(subtaskCtx) - if err != nil { - return err - } - err = r.CollectCommits(subtaskCtx) - if err != nil { - return err - } - opt := subtaskCtx.GetData().(*GitExtractorTaskData).Options - if !*opt.SkipCommitStat { - return r.CollectDiffLine(subtaskCtx) - } - return nil -} - -// Close resources -func (r *Libgit2RepoCollector) Close(ctx context.Context) error { - defer func() { - if r.cleanup != nil { - r.cleanup() - } - }() - return r.store.Close() -} - -// CountTags Count git tags subtask -func (r *Libgit2RepoCollector) CountTags(ctx context.Context) (int, error) { - tags, err := r.repo.Tags.List() - if err != nil { - return 0, errors.Convert(err) - } - return len(tags), nil -} - -// CountBranches count the number of branches in a git repo -func (r *Libgit2RepoCollector) CountBranches(ctx context.Context) (int, error) { - var branchIter *git.BranchIterator - branchIter, err := r.repo.NewBranchIterator(git.BranchAll) - if err != nil { - return 0, errors.Convert(err) - } - count := 0 - err = branchIter.ForEach(func(branch *git.Branch, branchType git.BranchType) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - if branch.IsBranch() || branch.IsRemote() { - isHead, _ := branch.IsHead() - if !isHead { - count++ - } - } - return nil - }) - return count, errors.Convert(err) -} - -// CountCommits count the number of commits in a git repo -func (r *Libgit2RepoCollector) CountCommits(ctx context.Context) (int, error) { - odb, err := r.repo.Odb() - if err != nil { - return 0, errors.Convert(err) - } - count := 0 - err = odb.ForEach(func(id *git.Oid) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - commit, e := r.repo.LookupCommit(id) - if e != nil && e.Error() != TypeNotMatchError { - return errors.Convert(e) - } - if commit != nil { - count++ - } - return nil - }) - return count, errors.Convert(err) -} - -// CollectTags Collect Tags data -func (r *Libgit2RepoCollector) CollectTags(subtaskCtx plugin.SubTaskContext) error { - return errors.Convert(r.repo.Tags.Foreach(func(name string, id *git.Oid) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - var err1 error - var tag *git.Tag - var tagCommit string - tag, err1 = r.repo.LookupTag(id) - if err1 != nil && err1.Error() != TypeNotMatchError { - return errors.Convert(err1) - } - if tag != nil { - tagCommit = tag.TargetId().String() - } else { - tagCommit = id.String() - } - r.logger.Debug("tagCommit: %s", tagCommit) - if tagCommit != "" { - ref := &code.Ref{ - DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, name)}, - RepoId: r.id, - Name: name, - CommitSha: tagCommit, - RefType: TAG, - } - err1 = r.store.Refs(ref) - if err1 != nil { - return err1 - } - subtaskCtx.IncProgress(1) - } - return nil - })) -} - -// CollectBranches Collect branch data -func (r *Libgit2RepoCollector) CollectBranches(subtaskCtx plugin.SubTaskContext) error { - var repoInter *git.BranchIterator - repoInter, err := r.repo.NewBranchIterator(git.BranchAll) - if err != nil { - return errors.Convert(err) - } - return errors.Convert(repoInter.ForEach(func(branch *git.Branch, branchType git.BranchType) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - if branch.IsBranch() || branch.IsRemote() { - name, err1 := branch.Name() - if err1 != nil && err1.Error() != TypeNotMatchError { - return err1 - } - var sha string - if oid := branch.Target(); oid != nil { - sha = oid.String() - } - ref := &code.Ref{ - DomainEntityExtended: domainlayer.DomainEntityExtended{Id: fmt.Sprintf("%s:%s", r.id, name)}, - RepoId: r.id, - Name: name, - CommitSha: sha, - RefType: BRANCH, - } - // commit sha may be an empty string, it's unexpected. - ref.IsDefault, err1 = branch.IsHead() - if err1 != nil && err1.Error() != TypeNotMatchError { - return err1 - } - err1 = r.store.Refs(ref) - if err1 != nil && err1.Error() != TypeNotMatchError { - return err1 - } - subtaskCtx.IncProgress(1) - return nil - } - return nil - })) -} - -// CollectCommits Collect data from each commit, we can also get the diff line -func (r *Libgit2RepoCollector) CollectCommits(subtaskCtx plugin.SubTaskContext) error { - taskOpts := subtaskCtx.GetData().(*GitExtractorTaskData).Options - opts, err := getDiffOpts() - if err != nil { - return err - } - db := subtaskCtx.GetDal() - components := make([]code.Component, 0) - err = db.All(&components, dal.From(components), dal.Where("repo_id= ?", r.id)) - if err != nil { - return err - } - componentMap := make(map[string]*regexp.Regexp) - for _, component := range components { - componentMap[component.Name] = regexp.MustCompile(component.PathRegex) - } - odb, err := errors.Convert01(r.repo.Odb()) - if err != nil { - return err - } - return errors.Convert(odb.ForEach(func(id *git.Oid) error { - select { - case <-subtaskCtx.GetContext().Done(): - return subtaskCtx.GetContext().Err() - default: - } - commit, err1 := r.repo.LookupCommit(id) - if err1 != nil && err1.Error() != TypeNotMatchError { - return errors.Convert(err1) - } - if commit == nil { - return nil - } - var parent *git.Commit - if commit.ParentCount() > 0 { - parent = commit.Parent(0) - // Skip calculating commit statistics when there are parent commits, but the first one cannot be fetched from the ODB. - // This usually happens during a shallow clone for incremental collection. Otherwise, we might end up overwriting - // the correct addition/deletion data in the database with an absurdly large addition number. - if parent == nil { - r.logger.Info("skip commit %s because it has no parent commit", commit.Id().String()) - return nil - } - } - commitSha := commit.Id().String() - r.logger.Debug("process commit: %s", commitSha) - c := &code.Commit{ - Sha: commitSha, - Message: commit.Message(), - } - author := commit.Author() - if author != nil { - c.AuthorName = author.Name - c.AuthorEmail = author.Email - c.AuthorId = author.Email - c.AuthoredDate = author.When - } - committer := commit.Committer() - if committer != nil { - c.CommitterName = committer.Name - c.CommitterEmail = committer.Email - c.CommitterId = committer.Email - c.CommittedDate = committer.When - } - err = r.storeParentCommits(commitSha, commit) - if err != nil { - return err - } - - if !*taskOpts.SkipCommitStat { - var stats *git.DiffStats - var addIncluded, delIncluded int - if stats, addIncluded, delIncluded, err = r.getDiffComparedToParent(taskOpts, c.Sha, commit, parent, opts, componentMap); err != nil { - return err - } - r.logger.Debug("state: %#+v\n", stats.Deletions()) - c.Additions += addIncluded - c.Deletions += delIncluded - } - - err = r.store.Commits(c) - if err != nil { - return err - } - repoCommit := &code.RepoCommit{ - RepoId: r.id, - CommitSha: c.Sha, - } - err = r.store.RepoCommits(repoCommit) - if err != nil { - return err - } - subtaskCtx.IncProgress(1) - return nil - })) -} - -func (r *Libgit2RepoCollector) storeParentCommits(commitSha string, commit *git.Commit) errors.Error { - var commitParents []*code.CommitParent - for i := uint(0); i < commit.ParentCount(); i++ { - parent := commit.Parent(i) - if parent != nil { - if parentId := parent.Id(); parentId != nil { - commitParents = append(commitParents, &code.CommitParent{ - CommitSha: commitSha, - ParentCommitSha: parentId.String(), - }) - } - } - } - return r.store.CommitParents(commitParents) -} - -func (r *Libgit2RepoCollector) getDiffComparedToParent(taskOpts *GitExtractorOptions, commitSha string, commit *git.Commit, parent *git.Commit, opts *git.DiffOptions, componentMap map[string]*regexp.Regexp) (*git.DiffStats, int, int, errors.Error) { - var err error - var parentTree, tree *git.Tree - if parent != nil { - parentTree, err = parent.Tree() - } - if err != nil { - return nil, 0, 0, errors.Convert(err) - } - tree, err = commit.Tree() - if err != nil { - return nil, 0, 0, errors.Convert(err) - } - var diff *git.Diff - diff, err = r.repo.DiffTreeToTree(parentTree, tree, opts) - if err != nil { - return nil, 0, 0, errors.Convert(err) - } - // build excluded extension set - excluded := map[string]struct{}{} - for _, ext := range taskOpts.ExcludeFileExtensions { - e := strings.ToLower(strings.TrimSpace(ext)) - if e == "" { - continue - } - excluded[e] = struct{}{} - } - if !*taskOpts.SkipCommitFiles { - err = r.storeCommitFilesFromDiff(commitSha, diff, componentMap, excluded) - if err != nil { - return nil, 0, 0, errors.Convert(err) - } - } - var stats *git.DiffStats - stats, err = diff.Stats() - if err != nil { - return nil, 0, 0, errors.Convert(err) - } - // calculate included totals with exclusions - addIncluded := 0 - delIncluded := 0 - if len(excluded) == 0 { - addIncluded = stats.Insertions() - delIncluded = stats.Deletions() - return stats, addIncluded, delIncluded, nil - } - _ = diff.ForEach(func(file git.DiffDelta, progress float64) (git.DiffForEachHunkCallback, error) { - // choose path to check based on delta status; for deletions use old path - pathForCheck := file.NewFile.Path - if file.Status == git.DeltaDeleted || pathForCheck == "" { - pathForCheck = file.OldFile.Path - } - lower := strings.ToLower(pathForCheck) - for ext := range excluded { - if strings.HasSuffix(lower, ext) { - // skip all lines for excluded files - return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) { - return func(line git.DiffLine) error { return nil }, nil - }, nil - } - } - return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) { - return func(line git.DiffLine) error { - if line.Origin == git.DiffLineAddition { - addIncluded += line.NumLines - } - if line.Origin == git.DiffLineDeletion { - delIncluded += line.NumLines - } - return nil - }, nil - }, nil - }, git.DiffDetailLines) - return stats, addIncluded, delIncluded, nil -} - -func (r *Libgit2RepoCollector) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, componentMap map[string]*regexp.Regexp, excluded map[string]struct{}) errors.Error { - var commitFile *code.CommitFile - var commitFileComponent *code.CommitFileComponent - var err error - err = diff.ForEach(func(file git.DiffDelta, progress float64) ( - git.DiffForEachHunkCallback, error) { - if commitFile != nil { - err = r.store.CommitFiles(commitFile) - if err != nil { - r.logger.Error(err, "CommitFiles error") - return nil, err - } - } - - // skip files by extension if configured - if len(excluded) > 0 { - pathForCheck := file.NewFile.Path - if file.Status == git.DeltaDeleted || pathForCheck == "" { - pathForCheck = file.OldFile.Path - } - lower := strings.ToLower(pathForCheck) - for ext := range excluded { - if strings.HasSuffix(lower, ext) { - // skip this file entirely - return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) { - return func(line git.DiffLine) error { return nil }, nil - }, nil - } - } - } - - commitFile = new(code.CommitFile) - commitFile.CommitSha = commitSha - // prefer new path; for deletions fall back to old path - commitFile.FilePath = file.NewFile.Path - if commitFile.FilePath == "" { - commitFile.FilePath = file.OldFile.Path - } - - // With some long path,the varchar(255) was not enough both ID and file_path - // So we use the hash to compress the path in ID and add length of file_path. - // Use commitSha and the sha256 of FilePath to create id - shaFilePath := sha256.New() - shaFilePath.Write([]byte(commitFile.FilePath)) - commitFile.Id = commitSha + ":" + hex.EncodeToString(shaFilePath.Sum(nil)) - - commitFileComponent = new(code.CommitFileComponent) - for component, reg := range componentMap { - if reg.MatchString(commitFile.FilePath) { - commitFileComponent.ComponentName = component - break - } - } - commitFileComponent.CommitFileId = commitFile.Id - if commitFileComponent.ComponentName == "" { - commitFileComponent.ComponentName = "Default" - } - return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) { - return func(line git.DiffLine) error { - if line.Origin == git.DiffLineAddition { - commitFile.Additions += line.NumLines - } - if line.Origin == git.DiffLineDeletion { - commitFile.Deletions += line.NumLines - } - return nil - }, nil - }, nil - }, git.DiffDetailLines) - if commitFileComponent != nil { - err = r.store.CommitFileComponents(commitFileComponent) - if err != nil { - r.logger.Error(err, "CommitFileComponents error") - } - } - if commitFile != nil { - err = r.store.CommitFiles(commitFile) - if err != nil { - r.logger.Error(err, "CommitFiles error") - } - } - return errors.Convert(err) -} - -// CollectDiffLine get line diff data from a specific branch -func (r *Libgit2RepoCollector) CollectDiffLine(subtaskCtx plugin.SubTaskContext) error { - //Using this subtask,we can get every line change in every commit. - //We maintain a snapshot structure to get which commit each deleted line belongs to - snapshot := make(map[string] /*file path*/ *models.FileBlame) - repo := r.repo - //step 1. get the reverse commit list - commitList := make([]git.Commit, 0) - // get current head commit sha, default is master branch - // check branch, if not master, checkout to branch's head - commitOid, err1 := repo.Head() - if err1 != nil && err1.Error() != TypeNotMatchError { - return errors.Convert(err1) - } - //get head commit object and add into commitList - commit, err1 := repo.LookupCommit(commitOid.Target()) - if err1 != nil && err1.Error() != TypeNotMatchError { - return errors.Convert(err1) - } - commitList = append(commitList, *commit) - // if current head has parents, get parent commitsha - for commit != nil && commit.ParentCount() > 0 { - pid := commit.ParentId(0) - commit, err1 = repo.LookupCommit(pid) - if err1 != nil && err1.Error() != TypeNotMatchError { - return errors.Convert(err1) - } - commitList = append(commitList, *commit) - } - // reverse commitList - for i, j := 0, len(commitList)-1; i < j; i, j = i+1, j-1 { - commitList[i], commitList[j] = commitList[j], commitList[i] - } - //step 2. get the diff of each commit - // for each commit, get the diff - for _, commitsha := range commitList { - curcommit, err := repo.LookupCommit(commitsha.Id()) - if err != nil { - return errors.Convert(err) - } - if curcommit.ParentCount() == 0 || curcommit.ParentCount() > 0 { - var parentTree, tree *git.Tree - tree, err = curcommit.Tree() - if err != nil { - return errors.Convert(err) - } - var diff *git.Diff - //FIXME error type convert - opts, err := git.DefaultDiffOptions() - opts.NotifyCallback = func(diffSoFar *git.Diff, delta git.DiffDelta, matchedPathSpec string) error { - return nil - } - if err != nil { - return errors.Convert(err) - } - if curcommit.ParentCount() > 0 { - parent := curcommit.Parent(0) - parentTree, err = parent.Tree() - } - diff, err = repo.DiffTreeToTree(parentTree, tree, &opts) - if err != nil { - return errors.Convert(err) - } - deleted := make(models.DiffLines, 0) - added := make(models.DiffLines, 0) - var lastFile string - lastFile = "" - err = diff.ForEach(func(file git.DiffDelta, progress float64) (git.DiffForEachHunkCallback, error) { - // if it doesn't exist in snapshot, create a new one - if _, ok := snapshot[file.OldFile.Path]; !ok { - fileBlame, err := models.NewFileBlame() - if err != nil { - r.logger.Info("Create FileBlame Error") - return nil, err - } - snapshot[file.OldFile.Path] = (*models.FileBlame)(fileBlame) - } - if lastFile == "" { - lastFile = file.NewFile.Path - } else if lastFile != file.NewFile.Path { - updateSnapshotFileBlame(curcommit, deleted, added, lastFile, snapshot) - // reset the deleted and added,last_file now is current file - deleted = make([]git.DiffLine, 0) - added = make([]git.DiffLine, 0) - lastFile = file.NewFile.Path - } - hunkNum := 0 - return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) { - hunkNum++ - return func(line git.DiffLine) error { - commitLineChange := &code.CommitLineChange{} - commitLineChange.CommitSha = curcommit.Id().String() - commitLineChange.ChangedType = line.Origin.String() - commitLineChange.LineNoNew = line.NewLineno - commitLineChange.LineNoOld = line.OldLineno - commitLineChange.OldFilePath = file.OldFile.Path - commitLineChange.NewFilePath = file.NewFile.Path - commitLineChange.HunkNum = hunkNum - commitLineChange.Id = curcommit.Id().String() + ":" + file.NewFile.Path + ":" + strconv.Itoa(line.OldLineno) + ":" + strconv.Itoa(line.NewLineno) - if line.Origin == git.DiffLineAddition { - added = append(added, line) - } else if line.Origin == git.DiffLineDeletion { - fb := snapshot[file.OldFile.Path] - l := fb.Find(line.OldLineno) - if l != nil && l.Value != nil { - temp := snapshot[file.OldFile.Path].Find(line.OldLineno) - commitLineChange.PrevCommit = temp.Value.(string) - } else { - r.logger.Info("err", file.OldFile.Path, line.OldLineno, curcommit.Id().String()) - } - deleted = append(deleted, line) - } - err = r.store.CommitLineChange(commitLineChange) - if err != nil { - return errors.Convert(err) - } - return nil - }, nil - }, nil - }, git.DiffDetailLines) - if err != nil { - return errors.Convert(err) - } - //finally,process the last file in diff - updateSnapshotFileBlame(curcommit, deleted, added, lastFile, snapshot) - } - } - r.logger.Info("line change collect success") - db := subtaskCtx.GetDal() - err := db.Delete(&code.RepoSnapshot{}, dal.Where("repo_id= ?", r.id)) - if err != nil { - return errors.Convert(err) - } - for fp := range snapshot { - temp := snapshot[fp] - count := 0 - for e := temp.Lines.Front(); e != nil; e = e.Next() { - count++ - snapshotLine := &code.RepoSnapshot{} - snapshotLine.RepoId = r.id - snapshotLine.LineNo = count - snapshotLine.CommitSha = e.Value.(string) - snapshotLine.FilePath = fp - err := r.store.RepoSnapshot(snapshotLine) - if err != nil { - r.logger.Info("error") - return err - } - } - - } - - r.logger.Info("collect snapshot finished") - return nil -} - -func updateSnapshotFileBlame(currentCommit *git.Commit, deleted models.DiffLines, added models.DiffLines, lastFile string, snapshot map[string]*models.FileBlame) { - sort.Sort(deleted) - for _, line := range deleted { - snapshot[lastFile].RemoveLine(line.OldLineno) - } - for _, line := range added { - snapshot[lastFile].AddLine(line.NewLineno, currentCommit.Id().String()) - } -} - -func getDiffOpts() (*git.DiffOptions, errors.Error) { - opts, err := git.DefaultDiffOptions() - if err != nil { - return nil, errors.Convert(err) - } - opts.NotifyCallback = func(diffSoFar *git.Diff, delta git.DiffDelta, matchedPathSpec string) error { - return nil - } - return &opts, nil -} diff --git a/backend/plugins/gitextractor/parser/taskdata.go b/backend/plugins/gitextractor/parser/taskdata.go index a8bb4510d..707c27b64 100644 --- a/backend/plugins/gitextractor/parser/taskdata.go +++ b/backend/plugins/gitextractor/parser/taskdata.go @@ -41,7 +41,6 @@ type GitExtractorOptions struct { PrivateKey string `json:"privateKey" mapstructure:"privateKey"` Passphrase string `json:"passphrase" mapstructure:"passphrase"` Proxy string `json:"proxy" mapstructure:"proxy"` - UseGoGit *bool `json:"useGoGit" mapstructure:"useGoGit"` SkipCommitStat *bool `json:"skipCommitStat" mapstructure:"skipCommitStat" comment:"skip all commit stat including added/deleted lines and commit files as well"` SkipCommitFiles *bool `json:"skipCommitFiles" mapstructure:"skipCommitFiles"` NoShallowClone bool `json:"noShallowClone" mapstructure:"noShallowClone"` @@ -49,8 +48,4 @@ type GitExtractorOptions struct { PluginName string `json:"pluginName" mapstructure:"pluginName,omitempty"` // Configured by upstream plugin (e.g., GitLab) to exclude file extensions from commit stats ExcludeFileExtensions []string `json:"excludeFileExtensions" mapstructure:"excludeFileExtensions"` - // UseGitCli selects the git command line collector. It is a temporary - // migration switch and will be removed once the library based collectors - // are gone. - UseGitCli bool `json:"useGitCli,omitempty" mapstructure:"useGitCli"` } diff --git a/backend/plugins/gitextractor/tasks/repo_cloner.go b/backend/plugins/gitextractor/tasks/repo_cloner.go index 63e58527b..0c218e9d8 100644 --- a/backend/plugins/gitextractor/tasks/repo_cloner.go +++ b/backend/plugins/gitextractor/tasks/repo_cloner.go @@ -69,15 +69,7 @@ func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error { if repoCloner.IsIncremental() { storage.SetIncrementalMode(repoCloner.IsIncremental()) } - // We have done comparison experiments for git2go and go-git, and the results show that git2go has better performance. - var repoCollector parser.RepoCollector - if taskData.Options.UseGitCli { - repoCollector, err = parser.NewGitcliRepoCollector(localDir, op.RepoId, storage, logger) - } else if *taskData.Options.UseGoGit { - repoCollector, err = parser.NewGogitRepoCollector(localDir, op.RepoId, storage, logger) - } else { - repoCollector, err = parser.NewLibgit2RepoCollector(localDir, op.RepoId, storage, logger) - } + repoCollector, err := parser.NewGitcliRepoCollector(localDir, op.RepoId, storage, logger) if err != nil { return err } diff --git a/backend/plugins/kiro/impl/table_info_test.go b/backend/plugins/kiro/impl/table_info_test.go index 90598fd1e..3291b12c3 100644 --- a/backend/plugins/kiro/impl/table_info_test.go +++ b/backend/plugins/kiro/impl/table_info_test.go @@ -24,10 +24,9 @@ import ( ) // The repo-wide plugins/table_info_test.go performs this same check for every -// plugin, but building that package requires gitextractor and therefore a -// specific libgit2. Running it here as well keeps the feedback local: a model -// added without registering it in GetTablesInfo fails immediately rather than -// only in CI. +// plugin. Running it here as well keeps the feedback local: a model added +// without registering it in GetTablesInfo fails immediately rather than only +// in CI. func TestKiroTableInfo(t *testing.T) { checker := unithelper.NewTableInfoChecker(unithelper.TableInfoCheckerConfig{}) checker.FeedIn("../models", Kiro{}.GetTablesInfo) diff --git a/backend/python/DevelopmentSetup.md b/backend/python/DevelopmentSetup.md index c831313c8..c171bd81d 100644 --- a/backend/python/DevelopmentSetup.md +++ b/backend/python/DevelopmentSetup.md @@ -16,7 +16,7 @@ limitations under the License. --> # Python prerequisites for `make e2e-test` -`make e2e-test` also requires a test database and, for some service tests, `libgit2`. See +`make e2e-test` also requires a test database and `git` for the service tests. See [`backend/test/Readme.md`](../test/Readme.md) for the full local E2E test flow. For the Python plugin tests to run properly, the following steps must be taken: diff --git a/backend/scripts/install-libgit2.sh b/backend/scripts/install-libgit2.sh deleted file mode 100755 index 7f6ced618..000000000 --- a/backend/scripts/install-libgit2.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eu - -LIBGIT2_VERSION=1.5.0 -LIBGIT2_SHA256=8de872a0f201b33d9522b817c92e14edb4efad18dae95cf156cf240b2efff93e -LIBGIT2_URL=https://github.com/libgit2/libgit2/archive/refs/tags/v${LIBGIT2_VERSION}.tar.gz - -current_version=$(pkg-config --modversion libgit2 2>/dev/null || true) -case "$current_version" in - 1.5.*) - echo "libgit2 $current_version is already installed" - exit 0 - ;; -esac - -if [ "$(id -u)" -ne 0 ]; then - echo "install-libgit2.sh must run as root" >&2 - exit 1 -fi - -build_dir=$(mktemp -d) -trap 'rm -rf "$build_dir"' EXIT HUP INT TERM - -export DEBIAN_FRONTEND=noninteractive - -# Debian releases are pulled from security.debian.org/deb.debian.org only -# while they have active (LTS) support. Once a release goes EOL, its -# packages move to archive.debian.org and the live mirrors 404. Repoint -# apt at the archive for releases we know are EOL so this keeps working -# without depending on the CI image being rebuilt on a newer base. -eol_codenames="bullseye" -codename=$(. /etc/os-release && echo "$VERSION_CODENAME") -case " $eol_codenames " in - *" $codename "*) - echo "Debian $codename is EOL; repointing apt sources at archive.debian.org" - for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list; do - [ -f "$f" ] || continue - sed -i \ - -e "s|http://deb.debian.org/debian|http://archive.debian.org/debian|g" \ - -e "s|http://security.debian.org/debian-security|http://archive.debian.org/debian-security|g" \ - "$f" - done - echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99no-check-valid-until - ;; -esac - -apt_update_and_install() { - apt-get update - apt-get install -y --no-install-recommends \ - ca-certificates \ - cmake \ - curl \ - gcc \ - libssh2-1-dev \ - libssl-dev \ - make \ - pkg-config \ - zlib1g-dev -} - -n=0 -until apt_update_and_install; do - n=$((n + 1)) - if [ "$n" -ge 3 ]; then - echo "apt-get update/install failed after $n attempts" >&2 - exit 1 - fi - echo "apt-get update/install failed, retrying ($n/3)..." >&2 - apt-get clean - sleep 5 -done - -archive="$build_dir/libgit2.tar.gz" -curl --fail --location --retry 3 "$LIBGIT2_URL" --output "$archive" -echo "$LIBGIT2_SHA256 $archive" | sha256sum --check --strict - -mkdir "$build_dir/source" -tar -xzf "$archive" -C "$build_dir/source" --strip-components=1 - -multiarch=$(gcc -print-multiarch) -libdir=lib -if [ -n "$multiarch" ]; then - libdir="lib/$multiarch" -fi - -cmake \ - -S "$build_dir/source" \ - -B "$build_dir/build" \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_TESTS=OFF \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_LIBDIR="$libdir" \ - -DCMAKE_INSTALL_PREFIX=/usr -cmake --build "$build_dir/build" --parallel "$(getconf _NPROCESSORS_ONLN)" -cmake --install "$build_dir/build" -ldconfig - -export PKG_CONFIG_PATH="/usr/$libdir/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" -installed_version=$(pkg-config --modversion libgit2) -if [ "$installed_version" != "$LIBGIT2_VERSION" ]; then - echo "expected libgit2 $LIBGIT2_VERSION, found $installed_version" >&2 - exit 1 -fi - -if [ -n "${GITHUB_ENV:-}" ]; then - echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH" >> "$GITHUB_ENV" -fi - -rm -rf /var/lib/apt/lists/* -echo "installed libgit2 $installed_version" diff --git a/backend/test/Readme.md b/backend/test/Readme.md index 87bfcfa30..70931816e 100644 --- a/backend/test/Readme.md +++ b/backend/test/Readme.md @@ -32,15 +32,8 @@ go run ./test/init.go make e2e-test ``` -The service E2E tests compile the `gitextractor` plugin, which links against `libgit2`. If `libgit2` is installed in -a non-standard location, expose it through `pkg-config` and add an rpath so the test binary can load the dynamic -library at runtime: - -```bash -export PKG_CONFIG_PATH=/path/to/libgit2/build -export CGO_LDFLAGS='-L/path/to/libgit2/build -Wl,-rpath,/path/to/libgit2/build' -make e2e-test -``` +The service E2E tests run the `gitextractor` plugin, which reads repositories +with the `git` command line. Make sure `git` (2.31 or newer) is installed. ### Planning-failure database regression test diff --git a/devops/docker/lake-builder/Dockerfile b/devops/docker/lake-builder/Dockerfile index 95cd0544e..64c95bbeb 100644 --- a/devops/docker/lake-builder/Dockerfile +++ b/devops/docker/lake-builder/Dockerfile @@ -14,51 +14,14 @@ # limitations under the License. # -# Debian 11 (bullseye) reached the end of its LTS period on 2026-08-31 and its -# security suite is no longer served, so `apt upgrade` fails there. All stages -# use Debian 12 (bookworm), matching the runtime image in backend/Dockerfile. -FROM --platform=linux/amd64 debian:bookworm AS debian-amd64 -RUN apt-get -y update && apt -y upgrade &&\ - apt-get install -y libssh2-1-dev libssl-dev zlib1g-dev - -# This stage only cross-builds libgit2, it does not need a Go toolchain. -FROM --platform=linux/amd64 debian:bookworm AS builder - -# Base dependencies -RUN apt-get -y update && apt -y upgrade &&\ - apt-get install -y ca-certificates wget gcc binutils libfindbin-libs-perl cmake libssh2-1-dev libssl-dev zlib1g-dev \ - gcc-x86-64-linux-gnu binutils-x86-64-linux-gnu - -COPY --from=debian-amd64 /usr/include /rootfs-amd64/usr/include -COPY --from=debian-amd64 /usr/lib/x86_64-linux-gnu /rootfs-amd64/usr/lib/x86_64-linux-gnu -COPY --from=debian-amd64 /lib/x86_64-linux-gnu /rootfs-amd64/lib/x86_64-linux-gnu - -## Build libs/headers needed for gitextractor plugin -RUN \ - mkdir -p /tmp/build/x86_64 && cd /tmp/build/x86_64 && \ - wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.5.0.tar.gz -O - | tar -xz && \ - cd libgit2-1.5.0 && \ - mkdir build && cd build && \ - cmake .. -DCMAKE_C_COMPILER=x86_64-linux-gnu-gcc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTS=OFF -DCMAKE_SYSROOT=/rootfs-amd64 -DCMAKE_INSTALL_PREFIX=/usr/local/deps/x86_64 &&\ - make -j install &&\ - mkdir -p /tmp/deps &&\ - cp *libgit2* /tmp/deps/ &&\ - cp -r ../include /tmp/deps/include - # Same Python line as the runtime image in backend/Dockerfile. FROM python:3.11-slim-bookworm RUN apt -y update && apt -y upgrade && apt -y install tzdata make tar curl gcc g++ pkg-config git \ - libssh2-1 zlib1g libffi-dev \ + zlib1g libffi-dev \ default-libmysqlclient-dev \ libpq-dev -# Install Libs/Headers from previous stage -COPY --from=builder /tmp/deps/*.so* /usr/lib/ -COPY --from=builder /tmp/deps/*.pc /usr/lib/x86_64-linux-gnu/pkgconfig/ -COPY --from=builder /tmp/deps/include/ /usr/include/ -ENV PKG_CONFIG_PATH=/usr/lib/x86_64-linux-gnu/pkgconfig - # Install Golang # The previous installer was fetched from `git.io`, which GitHub shut down in # 2022, so the image could no longer be rebuilt. Fetch the official archive diff --git a/env.example b/env.example index 68be8d6d2..9a4acb0dc 100755 --- a/env.example +++ b/env.example @@ -111,9 +111,6 @@ FORBID_REDIRECTION=false GITLAB_SERVER_COLLECT_ALL_USERS=true ########################## -# In plugin gitextractor, use go-git to collector repo's data -########################## -USE_GO_GIT_IN_GIT_EXTRACTOR=false # NOTE that COMMIT_FILES is part of the COMMIT_STAT SKIP_COMMIT_STAT=false SKIP_COMMIT_FILES=true
