This is an automated email from the ASF dual-hosted git repository. abeizn pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit ca2404c31ad4d92e95ff9882223bc6a654dd2aa5 Author: xgdyp <[email protected]> AuthorDate: Mon Jul 11 17:00:28 2022 +0800 fix: change migrationscript --- models/domainlayer/code/commit.go | 11 ++++ models/migrationscripts/register.go | 2 + .../updateSchemas20220711.go} | 59 ++++++++++++++-------- plugins/gitextractor/parser/repo.go | 31 ++++++++++-- 4 files changed, 79 insertions(+), 24 deletions(-) diff --git a/models/domainlayer/code/commit.go b/models/domainlayer/code/commit.go index 551e88aa..6d1ead84 100644 --- a/models/domainlayer/code/commit.go +++ b/models/domainlayer/code/commit.go @@ -50,8 +50,19 @@ type CommitFile struct { FilePath string `gorm:"primaryKey;type:varchar(255)"` Additions int Deletions int + Component string `gorm:"type:varchar(255)"` } func (CommitFile) TableName() string { return "commit_files" } + +type FileComponent struct { + RepoId string `gorm:"primaryKey;type:varchar(255)"` + Component string `gorm:"primaryKey;type:varchar(255)"` + PathRegex string `gorm:"type:varchar(255)"` +} + +func (FileComponent) TableName() string { + return "file_component" +} diff --git a/models/migrationscripts/register.go b/models/migrationscripts/register.go index 5294da8f..2f5351b3 100644 --- a/models/migrationscripts/register.go +++ b/models/migrationscripts/register.go @@ -22,6 +22,7 @@ import "github.com/apache/incubator-devlake/migration" // All return all the migration scripts of framework func All() []migration.Script { return []migration.Script{ + new(addFrameTables), new(renameStepToStage), new(addSubtasksField), @@ -29,5 +30,6 @@ func All() []migration.Script { new(renameTasksToPlan), new(addDomainTables), new(addTypeField), + new(updateSchemas20220711), } } diff --git a/models/domainlayer/code/commit.go b/models/migrationscripts/updateSchemas20220711.go similarity index 52% copy from models/domainlayer/code/commit.go copy to models/migrationscripts/updateSchemas20220711.go index 551e88aa..0268f5d0 100644 --- a/models/domainlayer/code/commit.go +++ b/models/migrationscripts/updateSchemas20220711.go @@ -15,33 +15,31 @@ See the License for the specific language governing permissions and limitations under the License. */ -package code +package migrationscripts import ( - "time" - + "context" "github.com/apache/incubator-devlake/models/common" + "gorm.io/gorm" ) -type Commit struct { - common.NoPKModel - Sha string `json:"sha" gorm:"primaryKey;type:varchar(40);comment:commit hash"` - Additions int `json:"additions" gorm:"comment:Added lines of code"` - Deletions int `json:"deletions" gorm:"comment:Deleted lines of code"` - DevEq int `json:"deveq" gorm:"comment:Merico developer equivalent from analysis engine"` - Message string - AuthorName string `gorm:"type:varchar(255)"` - AuthorEmail string `gorm:"type:varchar(255)"` - AuthoredDate time.Time - AuthorId string `gorm:"type:varchar(255)"` - CommitterName string `gorm:"type:varchar(255)"` - CommitterEmail string `gorm:"type:varchar(255)"` - CommittedDate time.Time - CommitterId string `gorm:"index;type:varchar(255)"` +//type CodeComponent20220711 struct { +// ComponentId string `gorm:"primaryKey;type:varchar(255)"` +// PathRegex string `gorm:"type:varchar(255)"` +//} +// +//func (CodeComponent20220711) TableName() string { +// return "code_component_20220711" +//} + +type FileComponent struct { + RepoId string `gorm:"primaryKey;type:varchar(255)"` + Component string `gorm:"primaryKey;type:varchar(255)"` + PathRegex string `gorm:"type:varchar(255)"` } -func (Commit) TableName() string { - return "commits" +func (FileComponent) TableName() string { + return "file_component" } type CommitFile struct { @@ -50,8 +48,29 @@ type CommitFile struct { FilePath string `gorm:"primaryKey;type:varchar(255)"` Additions int Deletions int + Component string `gorm:"type:varchar(255)"` } func (CommitFile) TableName() string { return "commit_files" } + +type updateSchemas20220711 struct{} + +func (*updateSchemas20220711) Up(ctx context.Context, db *gorm.DB) error { + + err := db.Migrator().AutoMigrate(FileComponent{}, CommitFile{}) + if err != nil { + return err + } + return nil + +} + +func (*updateSchemas20220711) Version() uint64 { + return 20220711122512 +} + +func (*updateSchemas20220711) Name() string { + return "file_component table" +} diff --git a/plugins/gitextractor/parser/repo.go b/plugins/gitextractor/parser/repo.go index 4d812e8f..84afbf3a 100644 --- a/plugins/gitextractor/parser/repo.go +++ b/plugins/gitextractor/parser/repo.go @@ -23,8 +23,10 @@ import ( "github.com/apache/incubator-devlake/models/domainlayer" "github.com/apache/incubator-devlake/models/domainlayer/code" "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/core/dal" "github.com/apache/incubator-devlake/plugins/gitextractor/models" git "github.com/libgit2/git2go/v33" + "regexp" ) type GitRepo struct { @@ -184,13 +186,25 @@ func (r *GitRepo) CollectBranches(subtaskCtx core.SubTaskContext) error { func (r *GitRepo) CollectCommits(subtaskCtx core.SubTaskContext) error { opts, err := getDiffOpts() + if err != nil { return err } + db := subtaskCtx.GetDal() + components := make([]code.FileComponent, 0) + err = db.All(&components, dal.From(components), dal.Where("repo_id= ?", r.id)) + if err != nil { + return err + } + componentMap := make(map[string]*regexp.Regexp) + for _, component := range components { + componentMap[component.Component] = regexp.MustCompile(component.PathRegex) + } odb, err := r.repo.Odb() if err != nil { return err } + return odb.ForEach(func(id *git.Oid) error { select { case <-subtaskCtx.GetContext().Done(): @@ -228,7 +242,7 @@ func (r *GitRepo) CollectCommits(subtaskCtx core.SubTaskContext) error { parent := commit.Parent(0) if parent != nil { var stats *git.DiffStats - if stats, err = r.getDiffComparedToParent(c.Sha, commit, parent, opts); err != nil { + if stats, err = r.getDiffComparedToParent(c.Sha, commit, parent, opts, componentMap); err != nil { return err } c.Additions += stats.Insertions() @@ -268,7 +282,7 @@ func (r *GitRepo) storeParentCommits(commitSha string, commit *git.Commit) error return r.store.CommitParents(commitParents) } -func (r *GitRepo) getDiffComparedToParent(commitSha string, commit *git.Commit, parent *git.Commit, opts *git.DiffOptions) (*git.DiffStats, error) { +func (r *GitRepo) getDiffComparedToParent(commitSha string, commit *git.Commit, parent *git.Commit, opts *git.DiffOptions, componentMap map[string]*regexp.Regexp) (*git.DiffStats, error) { var err error var parentTree, tree *git.Tree parentTree, err = parent.Tree() @@ -284,7 +298,7 @@ func (r *GitRepo) getDiffComparedToParent(commitSha string, commit *git.Commit, if err != nil { return nil, err } - err = r.storeCommitFilesFromDiff(commitSha, diff) + err = r.storeCommitFilesFromDiff(commitSha, diff, componentMap) if err != nil { return nil, err } @@ -296,7 +310,7 @@ func (r *GitRepo) getDiffComparedToParent(commitSha string, commit *git.Commit, return stats, nil } -func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff) error { +func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, componentMap map[string]*regexp.Regexp) error { var commitFile *code.CommitFile var err error err = diff.ForEach(func(file git.DiffDelta, progress float64) ( @@ -311,6 +325,15 @@ func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff) err commitFile = new(code.CommitFile) commitFile.CommitSha = commitSha commitFile.FilePath = file.NewFile.Path + for component, reg := range componentMap { + if reg.MatchString(commitFile.FilePath) { + commitFile.Component = component + break + } + } + if commitFile.Component == "" { + commitFile.Component = "Default" + } return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) { return func(line git.DiffLine) error { if line.Origin == git.DiffLineAddition {
