This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new ce3fdb88 feat: add snapshot table in gitextractor plugin (#3216)
ce3fdb88 is described below
commit ce3fdb8800d7ef7886a92f60f10c470da5174aeb
Author: xgdyp <[email protected]>
AuthorDate: Mon Oct 10 17:31:36 2022 +0800
feat: add snapshot table in gitextractor plugin (#3216)
* fix: fix conflict in gitextractor
* fix: change repo_snapshot table structure
* fix: change repo_snapshot structure in domainlayer
---
models/domainlayer/code/commit.go | 12 ++++++
.../migrationscripts/20220927_add_snapshot.go | 43 ++++++++++++++++------
models/migrationscripts/register.go | 1 +
plugins/gitextractor/gitextractor.go | 7 ++++
plugins/gitextractor/models/interface.go | 1 +
.../{interface.go => migrationscripts/register.go} | 17 +++------
plugins/gitextractor/parser/repo.go | 26 +++++++++++++
plugins/gitextractor/store/csv.go | 12 ++++++
plugins/gitextractor/store/database.go | 8 ++++
plugins/gitextractor/tasks/git_repo_collector.go | 16 ++++----
10 files changed, 112 insertions(+), 31 deletions(-)
diff --git a/models/domainlayer/code/commit.go
b/models/domainlayer/code/commit.go
index 13a7839b..1737fe4a 100644
--- a/models/domainlayer/code/commit.go
+++ b/models/domainlayer/code/commit.go
@@ -84,3 +84,15 @@ type CommitLineChange struct {
func (CommitLineChange) TableName() string {
return "commit_line_change"
}
+
+type RepoSnapshot struct {
+ common.NoPKModel
+ RepoId string `gorm:"primaryKey;type:varchar(255)"`
+ CommitSha string `gorm:"primaryKey;type:varchar(40);"`
+ FilePath string `gorm:"primaryKey;type:varchar(255);"`
+ LineNo int `gorm:"primaryKey;type:int;"`
+}
+
+func (RepoSnapshot) TableName() string {
+ return "repo_snapshot"
+}
diff --git a/plugins/gitextractor/models/interface.go
b/models/migrationscripts/20220927_add_snapshot.go
similarity index 50%
copy from plugins/gitextractor/models/interface.go
copy to models/migrationscripts/20220927_add_snapshot.go
index 9c264166..42edb81a 100644
--- a/plugins/gitextractor/models/interface.go
+++ b/models/migrationscripts/20220927_add_snapshot.go
@@ -15,20 +15,41 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package models
+package migrationscripts
import (
+ "context"
"github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/common"
+ "gorm.io/gorm"
)
-type Store interface {
- RepoCommits(repoCommit *code.RepoCommit) errors.Error
- Commits(commit *code.Commit) errors.Error
- Refs(ref *code.Ref) errors.Error
- CommitFiles(file *code.CommitFile) errors.Error
- CommitParents(pp []*code.CommitParent) errors.Error
- CommitFileComponents(commitFileComponent *code.CommitFileComponent)
errors.Error
- CommitLineChange(commitLineChange *code.CommitLineChange) errors.Error
- Close() errors.Error
+type RepoSnapshot struct {
+ common.NoPKModel
+ RepoId string `gorm:"primaryKey;type:varchar(255)"`
+ CommitSha string `gorm:"primaryKey;type:varchar(40);"`
+ FilePath string `gorm:"primaryKey;type:varchar(255);"`
+ LineNo int `gorm:"primaryKey;type:int;"`
+}
+
+func (RepoSnapshot) TableName() string {
+ return "repo_snapshot"
+}
+
+type addRepoSnapshot struct{}
+
+func (*addRepoSnapshot) Up(ctx context.Context, db *gorm.DB) errors.Error {
+ err := db.Migrator().AutoMigrate(RepoSnapshot{})
+ if err != nil {
+ return errors.Convert(err)
+ }
+ return nil
+}
+
+func (*addRepoSnapshot) Version() uint64 {
+ return 20221009111241
+}
+
+func (*addRepoSnapshot) Name() string {
+ return "add snapshot table"
}
diff --git a/models/migrationscripts/register.go
b/models/migrationscripts/register.go
index 1ef74d7f..31867c7d 100644
--- a/models/migrationscripts/register.go
+++ b/models/migrationscripts/register.go
@@ -48,5 +48,6 @@ func All() []migration.Script {
new(renamePipelineCommits),
new(commitLineChange),
new(modifyLeadTimeMinutes),
+ new(addRepoSnapshot),
}
}
diff --git a/plugins/gitextractor/gitextractor.go
b/plugins/gitextractor/gitextractor.go
index 5a3866e3..e15f7a1e 100644
--- a/plugins/gitextractor/gitextractor.go
+++ b/plugins/gitextractor/gitextractor.go
@@ -21,8 +21,10 @@ import (
"strings"
"github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/migration"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/gitextractor/models"
+
"github.com/apache/incubator-devlake/plugins/gitextractor/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/gitextractor/parser"
"github.com/apache/incubator-devlake/plugins/gitextractor/store"
"github.com/apache/incubator-devlake/plugins/gitextractor/tasks"
@@ -31,6 +33,7 @@ import (
var _ core.PluginMeta = (*GitExtractor)(nil)
var _ core.PluginTask = (*GitExtractor)(nil)
+var _ core.Migratable = (*GitExtractor)(nil)
type GitExtractor struct{}
@@ -78,6 +81,10 @@ func (plugin GitExtractor) Close(taskCtx core.TaskContext)
errors.Error {
return nil
}
+func (plugin GitExtractor) MigrationScripts() []migration.Script {
+ return migrationscripts.All()
+}
+
func (plugin GitExtractor) RootPkgPath() string {
return "github.com/apache/incubator-devlake/plugins/gitextractor"
}
diff --git a/plugins/gitextractor/models/interface.go
b/plugins/gitextractor/models/interface.go
index 9c264166..c4b235f7 100644
--- a/plugins/gitextractor/models/interface.go
+++ b/plugins/gitextractor/models/interface.go
@@ -30,5 +30,6 @@ type Store interface {
CommitParents(pp []*code.CommitParent) errors.Error
CommitFileComponents(commitFileComponent *code.CommitFileComponent)
errors.Error
CommitLineChange(commitLineChange *code.CommitLineChange) errors.Error
+ RepoSnapshot(snapshot *code.RepoSnapshot) errors.Error
Close() errors.Error
}
diff --git a/plugins/gitextractor/models/interface.go
b/plugins/gitextractor/models/migrationscripts/register.go
similarity index 58%
copy from plugins/gitextractor/models/interface.go
copy to plugins/gitextractor/models/migrationscripts/register.go
index 9c264166..ddb59c0a 100644
--- a/plugins/gitextractor/models/interface.go
+++ b/plugins/gitextractor/models/migrationscripts/register.go
@@ -15,20 +15,13 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package models
+package migrationscripts
import (
- "github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/migration"
)
-type Store interface {
- RepoCommits(repoCommit *code.RepoCommit) errors.Error
- Commits(commit *code.Commit) errors.Error
- Refs(ref *code.Ref) errors.Error
- CommitFiles(file *code.CommitFile) errors.Error
- CommitParents(pp []*code.CommitParent) errors.Error
- CommitFileComponents(commitFileComponent *code.CommitFileComponent)
errors.Error
- CommitLineChange(commitLineChange *code.CommitLineChange) errors.Error
- Close() errors.Error
+// All return all the migration scripts
+func All() []migration.Script {
+ return []migration.Script{}
}
diff --git a/plugins/gitextractor/parser/repo.go
b/plugins/gitextractor/parser/repo.go
index 8d3f3ffc..73361f41 100644
--- a/plugins/gitextractor/parser/repo.go
+++ b/plugins/gitextractor/parser/repo.go
@@ -508,6 +508,32 @@ func (r *GitRepo) CollectDiffLine(subtaskCtx
core.SubTaskContext) errors.Error {
updateSnapshotFileBlame(curcommit, deleted, added,
lastFile, snapshot)
}
}
+ r.logger.Info("line change collect success")
+ db := subtaskCtx.GetDal()
+ err := db.Delete(&code.RepoSnapshot{}, dal.Where("repo_id= ?", r.id))
+ if err != nil {
+ return errors.Convert(err)
+ }
+ for fp := range snapshot {
+ temp := snapshot[fp]
+ count := 0
+ for e := temp.Lines.Front(); e != nil; e = e.Next() {
+ count++
+ snapshotLine := &code.RepoSnapshot{}
+ snapshotLine.RepoId = r.id
+ snapshotLine.LineNo = count
+ snapshotLine.CommitSha = e.Value.(string)
+ snapshotLine.FilePath = fp
+ err := r.store.RepoSnapshot(snapshotLine)
+ if err != nil {
+ r.logger.Info("error")
+ return err
+ }
+ }
+
+ }
+
+ r.logger.Info("collect snapshot finished")
return nil
}
diff --git a/plugins/gitextractor/store/csv.go
b/plugins/gitextractor/store/csv.go
index 76cf0f2d..bafe2fc4 100644
--- a/plugins/gitextractor/store/csv.go
+++ b/plugins/gitextractor/store/csv.go
@@ -86,6 +86,7 @@ type CsvStore struct {
commitParentWriter *csvWriter
commitFileComponentWriter *csvWriter
commitLineChangeWriter *csvWriter
+ snapshotWriter *csvWriter
}
func NewCsvStore(dir string) (*CsvStore, errors.Error) {
@@ -125,6 +126,10 @@ func NewCsvStore(dir string) (*CsvStore, errors.Error) {
if err != nil {
return nil, errors.Convert(err)
}
+ s.snapshotWriter, err = newCsvWriter(filepath.Join(dir,
"repo_snapshot.csv"), code.RepoSnapshot{})
+ if err != nil {
+ return nil, errors.Convert(err)
+ }
return s, nil
}
@@ -152,6 +157,10 @@ func (c *CsvStore) CommitLineChange(commitLineChange
*code.CommitLineChange) err
return c.commitLineChangeWriter.Write(commitLineChange)
}
+func (c *CsvStore) RepoSnapshot(ss *code.RepoSnapshot) errors.Error {
+ return c.commitLineChangeWriter.Write(ss)
+}
+
func (c *CsvStore) CommitParents(pp []*code.CommitParent) errors.Error {
var err error
for _, p := range pp {
@@ -179,5 +188,8 @@ func (c *CsvStore) Close() errors.Error {
if c.commitParentWriter != nil {
c.commitParentWriter.Close()
}
+ if c.snapshotWriter != nil {
+ c.snapshotWriter.Close()
+ }
return nil
}
diff --git a/plugins/gitextractor/store/database.go
b/plugins/gitextractor/store/database.go
index d0c0c09c..f34212b3 100644
--- a/plugins/gitextractor/store/database.go
+++ b/plugins/gitextractor/store/database.go
@@ -100,6 +100,14 @@ func (d *Database)
CommitFileComponents(commitFileComponent *code.CommitFileComp
return batch.Add(commitFileComponent)
}
+func (d *Database) RepoSnapshot(snapshotElement *code.RepoSnapshot)
errors.Error {
+ batch, err := d.driver.ForType(reflect.TypeOf(snapshotElement))
+ if err != nil {
+ return err
+ }
+ return batch.Add(snapshotElement)
+}
+
func (d *Database) CommitLineChange(commitLineChange *code.CommitLineChange)
errors.Error {
batch, err := d.driver.ForType(reflect.TypeOf(commitLineChange))
if err != nil {
diff --git a/plugins/gitextractor/tasks/git_repo_collector.go
b/plugins/gitextractor/tasks/git_repo_collector.go
index 11742100..547cbb72 100644
--- a/plugins/gitextractor/tasks/git_repo_collector.go
+++ b/plugins/gitextractor/tasks/git_repo_collector.go
@@ -101,14 +101,6 @@ func getGitRepo(subTaskCtx core.SubTaskContext)
*parser.GitRepo {
return repo
}
-var CollectGitDiffLineMeta = core.SubTaskMeta{
- Name: "collectDiffLine",
- EntryPoint: CollectGitDiffLines,
- EnabledByDefault: false,
- Description: "collect git commit diff line into Domain Layer
Tables",
- DomainTypes: []string{core.DOMAIN_TYPE_CODE},
-}
-
var CollectGitCommitMeta = core.SubTaskMeta{
Name: "collectGitCommits",
EntryPoint: CollectGitCommits,
@@ -132,3 +124,11 @@ var CollectGitTagMeta = core.SubTaskMeta{
Description: "collect git tag into Domain Layer Tables",
DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
+
+var CollectGitDiffLineMeta = core.SubTaskMeta{
+ Name: "collectDiffLine",
+ EntryPoint: CollectGitDiffLines,
+ EnabledByDefault: false,
+ Description: "collect git commit diff line into Domain Layer
Tables",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
+}
