This is an automated email from the ASF dual-hosted git repository. kezhenxu94 pushed a commit to branch review in repository https://gitbox.apache.org/repos/asf/skywalking-eyes.git
commit 0719c7f28decb2ba7de459456c9bc51a546eb44e Author: kezhenxu94 <[email protected]> AuthorDate: Wed Dec 23 18:23:32 2020 +0800 Add new feature to review the pull request and suggest adding license headers --- .licenserc.yaml | 2 + license-eye/commands/header/check.go | 4 + license-eye/go.mod | 2 + license-eye/go.sum | 11 +- license-eye/pkg/header/config.go | 17 ++- license-eye/pkg/header/fix.go | 4 +- license-eye/pkg/header/fix_test.go | 4 +- license-eye/pkg/review/header.go | 274 +++++++++++++++++++++++++++++++++++ 8 files changed, 309 insertions(+), 9 deletions(-) diff --git a/.licenserc.yaml b/.licenserc.yaml index 819f2b1..d77578b 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -69,3 +69,5 @@ header: # `header` section is configurations for source codes license header. - 'NOTICE' - '**/assets/languages.yaml' - '**/assets/assets.gen.go' + + comment: on-failure # on what condition license-eye will comment on the pull request, `on-failure`, `always`, `never`. diff --git a/license-eye/commands/header/check.go b/license-eye/commands/header/check.go index 0578286..6f9e76f 100644 --- a/license-eye/commands/header/check.go +++ b/license-eye/commands/header/check.go @@ -22,6 +22,7 @@ import ( "github.com/apache/skywalking-eyes/license-eye/pkg" "github.com/apache/skywalking-eyes/license-eye/pkg/config" "github.com/apache/skywalking-eyes/license-eye/pkg/header" + "github.com/apache/skywalking-eyes/license-eye/pkg/review" "github.com/spf13/cobra" ) @@ -50,6 +51,9 @@ var CheckCommand = &cobra.Command{ logger.Log.Infoln(result.String()) if result.HasFailure() { + if err := review.Header(&result, &config); err != nil { + logger.Log.Warnln("Failed to create review comments", err) + } return result.Error() } diff --git a/license-eye/go.mod b/license-eye/go.mod index bba41da..096d2fb 100644 --- a/license-eye/go.mod +++ b/license-eye/go.mod @@ -4,7 +4,9 @@ go 1.13 require ( github.com/bmatcuk/doublestar/v2 v2.0.4 + github.com/google/go-github/v33 v33.0.0 github.com/sirupsen/logrus v1.7.0 github.com/spf13/cobra v1.1.1 + golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 ) diff --git a/license-eye/go.sum b/license-eye/go.sum index 59737cb..b1a492f 100644 --- a/license-eye/go.sum +++ b/license-eye/go.sum @@ -16,7 +16,6 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/apache/skywalking-eyes v0.0.0-20201221094504-6a06cc7bcc31 h1:Bc/I5QMp74IjLbTGlfAGabjjlFRr0nTlF5F04/WaE/k= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -57,11 +56,17 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github/v33 v33.0.0 h1:qAf9yP0qc54ufQxzwv+u9H0tiVOnPJxo0lI/JXqw3ZM= +github.com/google/go-github/v33 v33.0.0/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -192,6 +197,7 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -224,9 +230,11 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -282,6 +290,7 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= diff --git a/license-eye/pkg/header/config.go b/license-eye/pkg/header/config.go index d303dd4..5d4595b 100644 --- a/license-eye/pkg/header/config.go +++ b/license-eye/pkg/header/config.go @@ -29,11 +29,20 @@ import ( "github.com/bmatcuk/doublestar/v2" ) +type CommentOption string + +var ( + Always CommentOption = "always" + Never CommentOption = "never" + OnFailure CommentOption = "on-failure" +) + type ConfigHeader struct { - License string `yaml:"license"` - Pattern string `yaml:"pattern"` - Paths []string `yaml:"paths"` - PathsIgnore []string `yaml:"paths-ignore"` + License string `yaml:"license"` + Pattern string `yaml:"pattern"` + Paths []string `yaml:"paths"` + PathsIgnore []string `yaml:"paths-ignore"` + Comment CommentOption `yaml:"comment"` } // NormalizedLicense returns the normalized string of the license content, diff --git a/license-eye/pkg/header/fix.go b/license-eye/pkg/header/fix.go index 203de06..f5e061d 100644 --- a/license-eye/pkg/header/fix.go +++ b/license-eye/pkg/header/fix.go @@ -62,7 +62,7 @@ func InsertComment(file string, style *comments.CommentStyle, config *ConfigHead return err } - licenseHeader, err := generateLicenseHeader(style, config) + licenseHeader, err := GenerateLicenseHeader(style, config) if err != nil { return err } @@ -94,7 +94,7 @@ func rewriteContent(style *comments.CommentStyle, content []byte, licenseHeader ) } -func generateLicenseHeader(style *comments.CommentStyle, config *ConfigHeader) (string, error) { +func GenerateLicenseHeader(style *comments.CommentStyle, config *ConfigHeader) (string, error) { if err := style.Validate(); err != nil { return "", err } diff --git a/license-eye/pkg/header/fix_test.go b/license-eye/pkg/header/fix_test.go index c6f307f..57da4c4 100644 --- a/license-eye/pkg/header/fix_test.go +++ b/license-eye/pkg/header/fix_test.go @@ -57,7 +57,7 @@ func TestFix(t *testing.T) { for _, test := range tests { t.Run(test.filename, func(t *testing.T) { style := comments.FileCommentStyle(test.filename) - if c, err := generateLicenseHeader(style, config); err != nil || c != test.comments { + if c, err := GenerateLicenseHeader(style, config); err != nil || c != test.comments { t.Log("Actual:", c) t.Log("Expected:", test.comments) t.Logf("Middle:'%v'\n", style.Middle) @@ -213,7 +213,7 @@ echo 'Hello' | echo 'world!' } func getLicenseHeader(filename string, tError func(args ...interface{})) string { - s, err := generateLicenseHeader(comments.FileCommentStyle(filename), config) + s, err := GenerateLicenseHeader(comments.FileCommentStyle(filename), config) if err != nil { tError(err) } diff --git a/license-eye/pkg/review/header.go b/license-eye/pkg/review/header.go new file mode 100644 index 0000000..31dc559 --- /dev/null +++ b/license-eye/pkg/review/header.go @@ -0,0 +1,274 @@ +package review + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "regexp" + "strconv" + "strings" + + "github.com/apache/skywalking-eyes/license-eye/internal/logger" + "github.com/apache/skywalking-eyes/license-eye/pkg" + comments2 "github.com/apache/skywalking-eyes/license-eye/pkg/comments" + config2 "github.com/apache/skywalking-eyes/license-eye/pkg/config" + header2 "github.com/apache/skywalking-eyes/license-eye/pkg/header" + "github.com/google/go-github/v33/github" + "golang.org/x/oauth2" +) + +var ( + Identification = "license-eye hidden identification" + + gh *github.Client + ctx context.Context + + owner string + repo string + sha string + pr int + + requiredEnvVars = []string{ + "GITHUB_TOKEN", + "GITHUB_HEAD_REF", + "GITHUB_REPOSITORY", + "GITHUB_EVENT_NAME", + "GITHUB_EVENT_PATH", + } +) + +func init() { + if !IsPR() { + return + } + if !IsGHA() { + panic(fmt.Errorf(fmt.Sprintf( + `this must be run on GitHub Actions or you have to set the environment variables %v manually.`, requiredEnvVars, + ))) + } + + s, err := GetSha() + if err == nil { + logger.Log.Warnln("failed to get sha", err) + return + } + + sha = s + token := os.Getenv("GITHUB_TOKEN") + ref := os.Getenv("GITHUB_REF") + fullName := os.Getenv("GITHUB_REPOSITORY") + logger.Log.Debugln("ref:", ref, "; repo:", fullName, "; sha:", sha) + ownerRepo := strings.Split(fullName, "/") + if len(ownerRepo) != 2 { + logger.Log.Warnln("Length of ownerRepo is not 2", ownerRepo) + return + } + owner, repo = ownerRepo[0], ownerRepo[1] + matches := regexp.MustCompile(`refs/pull/(\d+)/merge`).FindStringSubmatch(ref) + if len(matches) < 1 { + logger.Log.Warnln("Length of ref < 1", matches) + return + } + prString := matches[1] // TODO: fault tolerant + if p, err := strconv.Atoi(prString); err == nil { + pr = p + } else { + logger.Log.Warnln("Failed to parse pull request number.", err) + return + } + + ctx = context.Background() + ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) + tc := oauth2.NewClient(ctx, ts) + gh = github.NewClient(tc) +} + +// Header reviews the license header, including suggestions on the pull request and an overview of the checks. +func Header(result *pkg.Result, config *config2.Config) error { + if !result.HasFailure() || !IsPR() || gh == nil { + return nil + } + + commentedFiles := make(map[string]bool) + for _, comment := range GetAllReviewsComments() { + decodeString := comment.GetBody() + if strings.Contains(decodeString, Identification) { + logger.Log.Debugln("Path:", comment.GetPath()) + commentedFiles[comment.GetPath()] = true + } + } + logger.Log.Debugln("CommentedFiles:", commentedFiles) + + s := "RIGHT" + l := 1 + + var comments []*github.DraftReviewComment + for _, changedFile := range GetChangedFiles() { + logger.Log.Debugln("ChangedFile:", changedFile.GetFilename()) + if commentedFiles[changedFile.GetFilename()] { + logger.Log.Debugln("ChangedFile was reviewed, skipping:", changedFile.GetFilename()) + continue + } + for _, invalidFile := range result.Failure { + if !strings.HasSuffix(invalidFile, changedFile.GetFilename()) { + continue + } + blob, _, err := gh.Git.GetBlob(ctx, owner, repo, changedFile.GetSHA()) + if err != nil { + logger.Log.Warnln("Failed to get blob:", changedFile.GetFilename(), changedFile.GetSHA()) + continue + } + header, err := header2.GenerateLicenseHeader(comments2.FileCommentStyle(changedFile.GetFilename()), &config.Header) + if err != nil { + logger.Log.Warnln("Failed to generate comment header:", changedFile.GetFilename()) + continue + } + decodeString, err := base64.StdEncoding.DecodeString(blob.GetContent()) + if err != nil { + logger.Log.Debugln("Failed to decode blob content:", err) + continue + } + body := "```suggestion\n" + header + strings.Split(string(decodeString), "\n")[0] + "\n```\n" + fmt.Sprintf(`<!-- %v -->`, Identification) + comments = append(comments, &github.DraftReviewComment{ + Path: changedFile.Filename, + Body: &body, + Side: &s, + Line: &l, + }) + } + } + + tryBestEffortToComment := func() error { + if err := doReview(result, comments); err != nil { + logger.Log.Warnln("Failed to create review comment, fallback to a plain comment:", err) + _ = doReview(result, nil) + return err + } + return nil + } + + if config.Header.Comment == header2.Always { + if err := tryBestEffortToComment(); err != nil { + return err + } + } else if config.Header.Comment == header2.OnFailure && len(comments) > 0 { + if err := tryBestEffortToComment(); err != nil { + return err + } + } + + return nil +} + +func doReview(result *pkg.Result, comments []*github.DraftReviewComment) error { + logger.Log.Debugln("Comments:", comments) + + c := Markdown(result) + e := "COMMENT" + if _, _, err := gh.PullRequests.CreateReview(ctx, owner, repo, pr, &github.PullRequestReviewRequest{ + CommitID: &sha, + Body: &c, + Event: &e, + Comments: comments, + }); err != nil { + return err + } + return nil +} + +// GetChangedFiles returns the changed files in this pull request. +func GetChangedFiles() []*github.CommitFile { + prsvc := gh.PullRequests + options := &github.ListOptions{Page: 1, PerPage: 100} + + var allFiles []*github.CommitFile + for files, response, err := prsvc.ListFiles(ctx, owner, repo, pr, options); err == nil; { + allFiles = append(allFiles, files...) + if response.NextPage <= options.Page { + break + } + options = &github.ListOptions{Page: response.NextPage, PerPage: options.PerPage} + } + return allFiles +} + +// GetAllReviewsComments returns all review comments of the pull request. +func GetAllReviewsComments() []*github.PullRequestComment { + prsvc := gh.PullRequests + options := &github.PullRequestListCommentsOptions{ListOptions: github.ListOptions{Page: 1, PerPage: 100}} + + var allComments []*github.PullRequestComment + for comments, response, err := prsvc.ListComments(ctx, owner, repo, pr, options); err == nil; { + allComments = append(allComments, comments...) + if response.NextPage <= options.Page { + break + } + options = &github.PullRequestListCommentsOptions{ + ListOptions: github.ListOptions{Page: response.NextPage, PerPage: options.PerPage}, + } + } + return allComments +} + +func IsGHA() bool { + for _, key := range requiredEnvVars { + if val := os.Getenv(key); val == "" { + return false + } + } + return true +} + +func IsPR() bool { + return os.Getenv("GITHUB_EVENT_NAME") == "pull_request" +} + +// TODO add fixing guide +func Markdown(result *pkg.Result) string { + return fmt.Sprintf(` +<!-- %s --> +[license-eye](https://github.com/apache/skywalking-eyes/tree/main/license-eye) has totally checked %d files. +| Valid | Invalid | Ignored | Fixed | +| --- | --- | --- | --- | +| %d | %d | %d | %d | +<details> + <summary>Click to see the invalid file list</summary> + + %v +</details> +`, + Identification, + len(result.Success)+len(result.Failure)+len(result.Ignored), + len(result.Success), + len(result.Failure), + len(result.Ignored), + len(result.Fixed), + "- "+strings.Join(result.Failure, "\n- "), + ) +} + +type Event struct { + PR github.PullRequest `json:"pull_request"` +} + +func GetSha() (string, error) { + filepath := os.Getenv("GITHUB_EVENT_PATH") + logger.Log.Debugln("GITHUB_EVENT_PATH: ", filepath) + if filepath == "" { + return "", fmt.Errorf("failed to get event path") + } + content, err := ioutil.ReadFile(filepath) + if err != nil { + return "", err + } + logger.Log.Debugln(filepath, "content:", string(content)) + + var event Event + if err = json.Unmarshal(content, &event); err != nil { + return "", err + } + return *event.PR.Head.SHA, nil +}
