This is an automated email from the ASF dual-hosted git repository. warren pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit a59ea07aee90f819487d33b29576cd36edde80af Author: Nddtfjiang <[email protected]> AuthorDate: Mon Jun 6 15:06:46 2022 +0000 feat: refdiff support pattern Add CaculateTagPattern Add CaculatePairList for CalculateIssuesDiff Add CalculateCommitsPairs for CalculateCommitsDiff Support tagsPattern tagsLimit and tagsOrder Nddtfjiang <[email protected]> --- plugins/refdiff/refdiff.go | 35 ++++++--- .../refdiff/tasks/ref_commit_diff_calculator.go | 70 ++++++++++++------ plugins/refdiff/tasks/ref_issue_diff_calculator.go | 33 +++++++-- plugins/refdiff/tasks/refdiff_task_data.go | 84 ++++++++++++++++++++++ 4 files changed, 187 insertions(+), 35 deletions(-) diff --git a/plugins/refdiff/refdiff.go b/plugins/refdiff/refdiff.go index 881feab1..9cfaa1f8 100644 --- a/plugins/refdiff/refdiff.go +++ b/plugins/refdiff/refdiff.go @@ -18,6 +18,8 @@ limitations under the License. package main import ( + "strconv" + "github.com/apache/incubator-devlake/plugins/refdiff/tasks" "github.com/apache/incubator-devlake/runner" "github.com/mitchellh/mapstructure" @@ -83,19 +85,34 @@ func main() { newRef := refdiffCmd.Flags().StringP("new-ref", "n", "", "new ref") oldRef := refdiffCmd.Flags().StringP("old-ref", "o", "", "old ref") + tagsPattern := refdiffCmd.Flags().StringP("tags-pattern", "p", "", "tags pattern") + tagsLimit := refdiffCmd.Flags().StringP("tags-limit", "l", "", "tags limit") + tagsOrder := refdiffCmd.Flags().StringP("tags-order", "d", "", "tags order") + _ = refdiffCmd.MarkFlagRequired("repo-id") - _ = refdiffCmd.MarkFlagRequired("new-ref") - _ = refdiffCmd.MarkFlagRequired("old-ref") + //_ = refdiffCmd.MarkFlagRequired("new-ref") + //_ = refdiffCmd.MarkFlagRequired("old-ref") refdiffCmd.Run = func(cmd *cobra.Command, args []string) { + tl, _ := strconv.Atoi(*tagsLimit) + pairs := make([]map[string]string, 0, 1) + if *newRef == "" && *oldRef == "" { + if *tagsPattern == "" { + panic("You must set at least one part of '-p' or '-n -o' for tagsPattern or newRef,oldRef") + } + } else { + pairs = append(pairs, map[string]string{ + "NewRef": *newRef, + "OldRef": *oldRef, + }) + } + runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ - "repoId": repoId, - "pairs": []map[string]string{ - { - "NewRef": *newRef, - "OldRef": *oldRef, - }, - }, + "repoId": repoId, + "pairs": pairs, + "tagsPattern": *tagsPattern, + "tagsLimit": tl, + "tagsOrder": *tagsOrder, }) } runner.RunCmd(refdiffCmd) diff --git a/plugins/refdiff/tasks/ref_commit_diff_calculator.go b/plugins/refdiff/tasks/ref_commit_diff_calculator.go index 17aa2bfd..69adb46f 100644 --- a/plugins/refdiff/tasks/ref_commit_diff_calculator.go +++ b/plugins/refdiff/tasks/ref_commit_diff_calculator.go @@ -27,14 +27,27 @@ import ( "gorm.io/gorm/clause" ) -func CalculateCommitsDiff(taskCtx core.SubTaskContext) error { +func CalculateCommitsPairs(taskCtx core.SubTaskContext) ([][4]string, error) { data := taskCtx.GetData().(*RefdiffTaskData) repoId := data.Options.RepoId pairs := data.Options.Pairs + tagsLimit := data.Options.TagsLimit db := taskCtx.GetDb() - ctx := taskCtx.GetContext() - logger := taskCtx.GetLogger() - insertCountLimitOfRefsCommitsDiff := int(65535 / reflect.ValueOf(code.RefsCommitsDiff{}).NumField()) + + rs, err := CaculateTagPattern(taskCtx) + if err != nil { + return [][4]string{}, err + } + if tagsLimit > rs.Len() { + tagsLimit = rs.Len() + } + + commitPairs := make([][4]string, 0, tagsLimit+len(pairs)) + for i := 1; i < tagsLimit; i++ { + commitPairs = append(commitPairs, [4]string{rs[i-1].CommitSha, rs[i].CommitSha, rs[i-1].Name, rs[i].Name}) + } + + // caculate pairs part // convert ref pairs into commit pairs ref2sha := func(refName string) (string, error) { ref := &code.Ref{} @@ -48,21 +61,37 @@ func CalculateCommitsDiff(taskCtx core.SubTaskContext) error { } return ref.CommitSha, nil } - commitPairs := make([][4]string, 0, len(pairs)) + for i, refPair := range pairs { // get new ref's commit sha newCommit, err := ref2sha(refPair.NewRef) if err != nil { - return fmt.Errorf("failed to load commit sha for NewRef on pair #%d: %w", i, err) + return [][4]string{}, fmt.Errorf("failed to load commit sha for NewRef on pair #%d: %w", i, err) } // get old ref's commit sha oldCommit, err := ref2sha(refPair.OldRef) if err != nil { - return fmt.Errorf("failed to load commit sha for OleRef on pair #%d: %w", i, err) + return [][4]string{}, fmt.Errorf("failed to load commit sha for OleRef on pair #%d: %w", i, err) } commitPairs = append(commitPairs, [4]string{newCommit, oldCommit, refPair.NewRef, refPair.OldRef}) } + return commitPairs, nil +} + +func CalculateCommitsDiff(taskCtx core.SubTaskContext) error { + data := taskCtx.GetData().(*RefdiffTaskData) + repoId := data.Options.RepoId + db := taskCtx.GetDb() + ctx := taskCtx.GetContext() + logger := taskCtx.GetLogger() + insertCountLimitOfRefsCommitsDiff := int(65535 / reflect.ValueOf(code.RefsCommitsDiff{}).NumField()) + + commitPairs, err := CalculateCommitsPairs(taskCtx) + if err != nil { + return err + } + commitNodeGraph := utils.NewCommitNodeGraph() // load commits from db @@ -90,7 +119,7 @@ func CalculateCommitsDiff(taskCtx core.SubTaskContext) error { commitNodeGraph.AddParent(commitParent.CommitSha, commitParent.ParentCommitSha) } - logger.Info("refdiff", fmt.Sprintf("Create a commit node graph with node count[%d]", commitNodeGraph.Size())) + logger.Info("Create a commit node graph with node count[%d]", commitNodeGraph.Size()) // calculate diffs for commits pairs and store them into database commitsDiff := &code.RefsCommitsDiff{} @@ -118,13 +147,10 @@ func CalculateCommitsDiff(taskCtx core.SubTaskContext) error { if commitsDiff.NewRefCommitSha == commitsDiff.OldRefCommitSha { // different refs might point to a same commit, it is ok logger.Info( - "refdiff", - fmt.Sprintf( - "skipping ref pair due to they are the same %s %s => %s", - commitsDiff.NewRefId, - commitsDiff.OldRefId, - commitsDiff.NewRefCommitSha, - ), + "skipping ref pair due to they are the same %s %s => %s", + commitsDiff.NewRefId, + commitsDiff.OldRefId, + commitsDiff.NewRefCommitSha, ) continue } @@ -140,7 +166,7 @@ func CalculateCommitsDiff(taskCtx core.SubTaskContext) error { // sql limit placeholders count only 65535 if commitsDiff.SortingIndex%insertCountLimitOfRefsCommitsDiff == 0 { - logger.Info("refdiff", fmt.Sprintf("commitsDiffs count in limited[%d] index[%d]--exec and clean", len(commitsDiffs), commitsDiff.SortingIndex)) + logger.Info("commitsDiffs count in limited[%d] index[%d]--exec and clean", len(commitsDiffs), commitsDiff.SortingIndex) err = db.Clauses(clause.OnConflict{DoNothing: true}).Create(commitsDiffs).Error if err != nil { return err @@ -152,20 +178,20 @@ func CalculateCommitsDiff(taskCtx core.SubTaskContext) error { } if len(commitsDiffs) > 0 { - logger.Info("refdiff", fmt.Sprintf("insert data count [%d]", len(commitsDiffs))) + logger.Info("insert data count [%d]", len(commitsDiffs)) err = db.Clauses(clause.OnConflict{DoNothing: true}).Create(commitsDiffs).Error if err != nil { return err } } - logger.Info("refdiff", fmt.Sprintf( - "total %d commits of difference found between %s and %s(total:%d)", + logger.Info( + "total %d commits of difference found between [new][%s] and [old][%s(total:%d)]", newCount, - commitsDiff.NewRefCommitSha, - commitsDiff.OldRefCommitSha, + commitsDiff.NewRefId, + commitsDiff.OldRefId, oldCount, - )) + ) taskCtx.IncProgress(1) } return nil diff --git a/plugins/refdiff/tasks/ref_issue_diff_calculator.go b/plugins/refdiff/tasks/ref_issue_diff_calculator.go index a164055b..3a80482d 100644 --- a/plugins/refdiff/tasks/ref_issue_diff_calculator.go +++ b/plugins/refdiff/tasks/ref_issue_diff_calculator.go @@ -26,15 +26,40 @@ import ( "github.com/apache/incubator-devlake/plugins/helper" ) -func CalculateIssuesDiff(taskCtx core.SubTaskContext) error { +func CaculatePairList(taskCtx core.SubTaskContext) ([][2]string, error) { data := taskCtx.GetData().(*RefdiffTaskData) repoId := data.Options.RepoId pairs := data.Options.Pairs + tagsLimit := data.Options.TagsLimit + + rs, err := CaculateTagPattern(taskCtx) + if err != nil { + return [][2]string{}, err + } + if tagsLimit > rs.Len() { + tagsLimit = rs.Len() + } + + pairList := make([][2]string, 0, tagsLimit+len(pairs)) + for i := 1; i < tagsLimit; i++ { + pairList = append(pairList, [2]string{fmt.Sprintf("%s:%s", repoId, rs[i-1].Id), fmt.Sprintf("%s:%s", repoId, rs[i].Id)}) + } + + for _, pair := range pairs { + pairList = append(pairList, [2]string{fmt.Sprintf("%s:%s", repoId, pair.NewRef), fmt.Sprintf("%s:%s", repoId, pair.OldRef)}) + } + + return pairList, nil +} + +func CalculateIssuesDiff(taskCtx core.SubTaskContext) error { + data := taskCtx.GetData().(*RefdiffTaskData) + repoId := data.Options.RepoId db := taskCtx.GetDb() // use to calculate progress - pairList := make([][2]string, len(pairs)) - for i, pair := range pairs { - pairList[i] = [2]string{fmt.Sprintf("%s:%s", repoId, pair.NewRef), fmt.Sprintf("%s:%s", repoId, pair.OldRef)} + pairList, err := CaculatePairList(taskCtx) + if err != nil { + return err } cursor, err := db.Table("refs_commits_diffs"). Joins( diff --git a/plugins/refdiff/tasks/refdiff_task_data.go b/plugins/refdiff/tasks/refdiff_task_data.go index 9b53c88a..c2b364fc 100644 --- a/plugins/refdiff/tasks/refdiff_task_data.go +++ b/plugins/refdiff/tasks/refdiff_task_data.go @@ -18,13 +18,23 @@ limitations under the License. package tasks import ( + "fmt" + "regexp" + "sort" "time" + + "github.com/apache/incubator-devlake/models/domainlayer/code" + "github.com/apache/incubator-devlake/plugins/core" ) type RefdiffOptions struct { RepoId string Tasks []string `json:"tasks,omitempty"` Pairs []RefPair + + TagsPattern string + TagsLimit int + TagsOrder string } type RefdiffTaskData struct { @@ -36,3 +46,77 @@ type RefPair struct { NewRef string OldRef string } + +type Refs []code.Ref +type RefsAlphabetically Refs +type RefsReverseAlphabetically Refs + +func (rs Refs) Len() int { + return len(rs) +} + +func (rs RefsAlphabetically) Len() int { + return len(rs) +} + +func (rs RefsAlphabetically) Less(i, j int) bool { + return rs[i].Id < rs[j].Id +} + +func (rs RefsAlphabetically) Swap(i, j int) { + rs[i], rs[j] = rs[j], rs[i] +} + +func (rs RefsReverseAlphabetically) Len() int { + return len(rs) +} + +func (rs RefsReverseAlphabetically) Less(i, j int) bool { + return rs[i].Id > rs[j].Id +} + +func (rs RefsReverseAlphabetically) Swap(i, j int) { + rs[i], rs[j] = rs[j], rs[i] +} + +func CaculateTagPattern(taskCtx core.SubTaskContext) (Refs, error) { + rs := Refs{} + data := taskCtx.GetData().(*RefdiffTaskData) + tagsPattern := data.Options.TagsPattern + tagsOrder := data.Options.TagsOrder + db := taskCtx.GetDb() + + // caculate Pattern part + if data.Options.TagsPattern == "" || data.Options.TagsLimit <= 1 { + return rs, nil + } + rows, err := db.Model(&code.Ref{}).Order("created_date desc").Rows() + if err != nil { + return rs, err + } + defer rows.Next() + r, err := regexp.Compile(tagsPattern) + if err != nil { + return rs, fmt.Errorf("unable to parse: %s\r\n%s", tagsPattern, err.Error()) + } + for rows.Next() { + var ref code.Ref + err = db.ScanRows(rows, &ref) + if err != nil { + return rs, err + } + + if ok := r.Match([]byte(ref.Id)); ok { + rs = append(rs, ref) + } + } + switch tagsOrder { + case "alphabetically": + sort.Sort(RefsAlphabetically(rs)) + case "reverse alphabetically": + sort.Sort(RefsReverseAlphabetically(rs)) + default: + break + } + return rs, nil +}
