This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 59e48852 GithubPlugin: Pass config in taskCtx instead of db models
(#2192)
59e48852 is described below
commit 59e48852f0d62042d9e06e87cf396ab325e0c8da
Author: likyh <[email protected]>
AuthorDate: Thu Jun 16 14:37:22 2022 +0800
GithubPlugin: Pass config in taskCtx instead of db models (#2192)
* Pass config in taskCtx instead of db models
* prepare migrate script init_schema for the coming PR which append
connectionId
* use inject instead of use config.GetConfig in migration script
Co-authored-by: linyh <[email protected]>
---
.env.example | 2 -
plugins/github/README.md | 13 +-
plugins/github/github.go | 167 +++++----------------
plugins/github/{github.go => impl/impl.go} | 67 +++++----
plugins/github/models/connection.go | 1 +
.../migrationscripts/archived/commit_stat.go | 7 +-
.../models/migrationscripts/archived/issue.go | 2 +
.../migrationscripts/archived/pull_request.go | 2 +-
.../models/migrationscripts/archived/repo.go | 6 +-
.../github/models/migrationscripts/init_schema.go | 54 -------
.../migrationscripts/init_schema_20220610.go | 139 +++++++++++++++++
.../migrationscripts/updateSchemas20220509.go | 86 -----------
.../migrationscripts/updateSchemas20220524.go | 54 -------
.../migrationscripts/updateSchemas20220608.go | 81 ----------
plugins/github/tasks/comment_collector.go | 3 +-
plugins/github/tasks/pr_issue_enricher.go | 2 +-
plugins/github/tasks/task_data.go | 11 +-
17 files changed, 242 insertions(+), 455 deletions(-)
diff --git a/.env.example b/.env.example
index 0ceca13e..80dc9e36 100644
--- a/.env.example
+++ b/.env.example
@@ -80,8 +80,6 @@ FEISHU_ENDPOINT=https://open.feishu.cn/open-apis/vc/v1/
########################
# GitHub configuration has been migrated into DB #
-# FIXME It's very special because no defined in config, so it will be deleted
in next PR
-GITHUB_PR_BODY_CLOSE_PATTERN='(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\s]*.*(((and
)?(#|https:\/\/github.com\/%s\/%s\/issues\/)\d+[ ]*)+)'
# FIXME this config use in refdiff
GITHUB_PR_TITLE_PATTERN='.*\(#(\d+)\)'
diff --git a/plugins/github/README.md b/plugins/github/README.md
index 0e8660f7..847233ce 100644
--- a/plugins/github/README.md
+++ b/plugins/github/README.md
@@ -65,8 +65,16 @@ Enter a valid proxy server address on your Network, e.g.
`http://your-proxy-serv
Click **Save Settings** to update additional settings.
### Regular Expression Configuration
-Define regex pattern in .env
-- GITHUB_PR_BODY_CLOSE_PATTERN: Define key word to associate issue in pr body,
please check the example in .env.example
+Define regex pattern in request options
+- prType: Define key word to associate issue in pr body
+- prComponent: Define key word to associate issue in pr body
+- prBodyClosePattern: Define key word to associate issue in pr body
+- issueSeverity: Define key word to associate issue in pr body
+- issuePriority: Define key word to associate issue in pr body
+- issueComponent: Define key word to associate issue in pr body
+- issueTypeBug: Define key word to associate issue in pr body
+- issueTypeIncident: Define key word to associate issue in pr body
+- issueTypeRequirement: Define key word to associate issue in pr body
## Sample Request
In order to collect data, you have to compose a JSON looks like following one,
and send it by selecting `Advanced Mode` on `Create Pipeline Run` page:
@@ -79,6 +87,7 @@ In order to collect data, you have to compose a JSON looks
like following one, a
"options": {
"repo": "lake",
"owner": "merico-dev"
+ // add more config such as prType if necessary.
}
}
]
diff --git a/plugins/github/github.go b/plugins/github/github.go
index 93ddf23a..d189912f 100644
--- a/plugins/github/github.go
+++ b/plugins/github/github.go
@@ -18,148 +18,49 @@ limitations under the License.
package main // must be main for plugin entry point
import (
- "fmt"
- "github.com/apache/incubator-devlake/migration"
- "github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/plugins/github/api"
- "github.com/apache/incubator-devlake/plugins/github/models"
-
"github.com/apache/incubator-devlake/plugins/github/models/migrationscripts"
- "github.com/apache/incubator-devlake/plugins/github/tasks"
+ "github.com/apache/incubator-devlake/plugins/github/impl"
"github.com/apache/incubator-devlake/runner"
- "github.com/mitchellh/mapstructure"
"github.com/spf13/cobra"
- "github.com/spf13/viper"
- "gorm.io/gorm"
)
-var _ core.PluginMeta = (*Github)(nil)
-var _ core.PluginInit = (*Github)(nil)
-var _ core.PluginTask = (*Github)(nil)
-var _ core.PluginApi = (*Github)(nil)
-var _ core.Migratable = (*Github)(nil)
-
-type Github struct{}
-
-func (plugin Github) Init(config *viper.Viper, logger core.Logger, db
*gorm.DB) error {
- api.Init(config, logger, db)
- return nil
-}
-
-func (plugin Github) Description() string {
- return "To collect and enrich data from GitHub"
-}
-
-func (plugin Github) SubTaskMetas() []core.SubTaskMeta {
- return []core.SubTaskMeta{
- tasks.CollectApiRepoMeta,
- tasks.ExtractApiRepoMeta,
- tasks.CollectApiIssuesMeta,
- tasks.ExtractApiIssuesMeta,
- tasks.CollectApiPullRequestsMeta,
- tasks.ExtractApiPullRequestsMeta,
- tasks.CollectApiCommentsMeta,
- tasks.ExtractApiCommentsMeta,
- tasks.CollectApiEventsMeta,
- tasks.ExtractApiEventsMeta,
- tasks.CollectApiPullRequestCommitsMeta,
- tasks.ExtractApiPullRequestCommitsMeta,
- tasks.CollectApiPullRequestReviewsMeta,
- tasks.ExtractApiPullRequestReviewsMeta,
- tasks.CollectApiCommitsMeta,
- tasks.ExtractApiCommitsMeta,
- tasks.CollectApiCommitStatsMeta,
- tasks.ExtractApiCommitStatsMeta,
- tasks.EnrichPullRequestIssuesMeta,
- tasks.ConvertRepoMeta,
- tasks.ConvertIssuesMeta,
- tasks.ConvertCommitsMeta,
- tasks.ConvertIssueLabelsMeta,
- tasks.ConvertPullRequestCommitsMeta,
- tasks.ConvertPullRequestsMeta,
- tasks.ConvertPullRequestLabelsMeta,
- tasks.ConvertPullRequestIssuesMeta,
- tasks.ConvertUsersMeta,
- tasks.ConvertIssueCommentsMeta,
- tasks.ConvertPullRequestCommentsMeta,
- }
-}
-
-func (plugin Github) PrepareTaskData(taskCtx core.TaskContext, options
map[string]interface{}) (interface{}, error) {
- var op tasks.GithubOptions
- err := mapstructure.Decode(options, &op)
- if err != nil {
- return nil, err
- }
- if op.Owner == "" {
- return nil, fmt.Errorf("owner is required for GitHub execution")
- }
- if op.Repo == "" {
- return nil, fmt.Errorf("repo is required for GitHub execution")
- }
-
- // find the only github now
- // FIXME to query by connection id when multi connection support
- connection := &models.GithubConnection{}
- err = taskCtx.GetDb().First(connection, "name = ?", `GitHub`).Error
- if err != nil {
- return err, nil
- }
-
- apiClient, err := tasks.CreateApiClient(taskCtx, connection)
- if err != nil {
- return nil, err
- }
-
- return &tasks.GithubTaskData{
- Options: &op,
- ApiClient: apiClient,
- }, nil
-}
-
-func (plugin Github) RootPkgPath() string {
- return "github.com/apache/incubator-devlake/plugins/github"
-}
-
-func (plugin Github) MigrationScripts() []migration.Script {
- return []migration.Script{
- new(migrationscripts.InitSchemas),
new(migrationscripts.UpdateSchemas20220509),
- new(migrationscripts.UpdateSchemas20220524),
new(migrationscripts.UpdateSchemas20220608),
- }
-}
-
-func (plugin Github) ApiResources()
map[string]map[string]core.ApiResourceHandler {
- return map[string]map[string]core.ApiResourceHandler{
- "test": {
- "POST": api.TestConnection,
- },
- "connections": {
- "POST": api.PostConnections,
- "GET": api.ListConnections,
- },
- "connections/:connectionId": {
- "GET": api.GetConnection,
- "PATCH": api.PatchConnection,
- "DELETE": api.DeleteConnection,
- },
- }
-}
-
// Export a variable named PluginEntry for Framework to search and load
-var PluginEntry Github //nolint
+var PluginEntry impl.Github //nolint
// standalone mode for debugging
func main() {
- githubCmd := &cobra.Command{Use: "github"}
- owner := githubCmd.Flags().StringP("owner", "o", "", "github owner")
- repo := githubCmd.Flags().StringP("repo", "r", "", "github repo")
- _ = githubCmd.MarkFlagRequired("owner")
- _ = githubCmd.MarkFlagRequired("repo")
-
- githubCmd.Run = func(cmd *cobra.Command, args []string) {
+ cmd := &cobra.Command{Use: "github"}
+ connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "github
connection id")
+ owner := cmd.Flags().StringP("owner", "o", "", "github owner")
+ repo := cmd.Flags().StringP("repo", "r", "", "github repo")
+ _ = cmd.MarkFlagRequired("connectionId")
+ _ = cmd.MarkFlagRequired("owner")
+ _ = cmd.MarkFlagRequired("repo")
+
+ prType := cmd.Flags().String("prType", "type/(.*)$", "pr type")
+ prComponent := cmd.Flags().String("prComponent", "component/(.*)$", "pr
component")
+ prBodyClosePattern := cmd.Flags().String("prBodyClosePattern",
"(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and
)?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)", "pr body close
pattern")
+ issueSeverity := cmd.Flags().String("issueSeverity", "severity/(.*)$",
"issue severity")
+ issuePriority := cmd.Flags().String("issuePriority",
"^(highest|high|medium|low)$", "issue priority")
+ issueComponent := cmd.Flags().String("issueComponent",
"component/(.*)$", "issue component")
+ issueTypeBug := cmd.Flags().String("issueTypeBug",
"^(bug|failure|error)$", "issue type bug")
+ issueTypeIncident := cmd.Flags().String("issueTypeIncident", "", "issue
type incident")
+ issueTypeRequirement := cmd.Flags().String("issueTypeRequirement",
"^(feat|feature|proposal|requirement)$", "issue type requirement")
+
+ cmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- "owner": *owner,
- "repo": *repo,
+ "connectionId": *connectionId,
+ "owner": *owner,
+ "repo": *repo,
+ "prType": *prType,
+ "prComponent": *prComponent,
+ "prBodyClosePattern": *prBodyClosePattern,
+ "issueSeverity": *issueSeverity,
+ "issuePriority": *issuePriority,
+ "issueComponent": *issueComponent,
+ "issueTypeBug": *issueTypeBug,
+ "issueTypeIncident": *issueTypeIncident,
+ "issueTypeRequirement": *issueTypeRequirement,
})
}
- runner.RunCmd(githubCmd)
+ runner.RunCmd(cmd)
}
diff --git a/plugins/github/github.go b/plugins/github/impl/impl.go
similarity index 77%
copy from plugins/github/github.go
copy to plugins/github/impl/impl.go
index 93ddf23a..b4f1ff21 100644
--- a/plugins/github/github.go
+++ b/plugins/github/impl/impl.go
@@ -15,7 +15,7 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package main // must be main for plugin entry point
+package impl
import (
"fmt"
@@ -25,9 +25,8 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/github/tasks"
- "github.com/apache/incubator-devlake/runner"
+ "github.com/apache/incubator-devlake/plugins/helper"
"github.com/mitchellh/mapstructure"
- "github.com/spf13/cobra"
"github.com/spf13/viper"
"gorm.io/gorm"
)
@@ -96,11 +95,44 @@ func (plugin Github) PrepareTaskData(taskCtx
core.TaskContext, options map[strin
if op.Repo == "" {
return nil, fmt.Errorf("repo is required for GitHub execution")
}
+ if op.PrType == "" {
+ op.PrType = "type/(.*)$"
+ }
+ if op.PrComponent == "" {
+ op.PrComponent = "component/(.*)$"
+ }
+ if op.PrBodyClosePattern == "" {
+ op.PrBodyClosePattern =
"(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and
)?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)"
+ }
+ if op.IssueSeverity == "" {
+ op.IssueSeverity = "severity/(.*)$"
+ }
+ if op.IssuePriority == "" {
+ op.IssuePriority = "^(highest|high|medium|low)$"
+ }
+ if op.IssueComponent == "" {
+ op.IssueComponent = "component/(.*)$"
+ }
+ if op.IssueTypeBug == "" {
+ op.IssueTypeBug = "^(bug|failure|error)$"
+ }
+ if op.IssueTypeIncident == "" {
+ op.IssueTypeIncident = ""
+ }
+ if op.IssueTypeRequirement == "" {
+ op.IssueTypeRequirement =
"^(feat|feature|proposal|requirement)$"
+ }
- // find the only github now
- // FIXME to query by connection id when multi connection support
+ // find the needed GitHub now
+ if op.ConnectionId == 0 {
+ return nil, fmt.Errorf("connectionId is invalid")
+ }
+ connectionHelper := helper.NewConnectionHelper(
+ taskCtx,
+ nil,
+ )
connection := &models.GithubConnection{}
- err = taskCtx.GetDb().First(connection, "name = ?", `GitHub`).Error
+ err = connectionHelper.FirstById(connection, op.ConnectionId)
if err != nil {
return err, nil
}
@@ -122,8 +154,7 @@ func (plugin Github) RootPkgPath() string {
func (plugin Github) MigrationScripts() []migration.Script {
return []migration.Script{
- new(migrationscripts.InitSchemas),
new(migrationscripts.UpdateSchemas20220509),
- new(migrationscripts.UpdateSchemas20220524),
new(migrationscripts.UpdateSchemas20220608),
+ new(migrationscripts.InitSchemas),
}
}
@@ -143,23 +174,3 @@ func (plugin Github) ApiResources()
map[string]map[string]core.ApiResourceHandle
},
}
}
-
-// Export a variable named PluginEntry for Framework to search and load
-var PluginEntry Github //nolint
-
-// standalone mode for debugging
-func main() {
- githubCmd := &cobra.Command{Use: "github"}
- owner := githubCmd.Flags().StringP("owner", "o", "", "github owner")
- repo := githubCmd.Flags().StringP("repo", "r", "", "github repo")
- _ = githubCmd.MarkFlagRequired("owner")
- _ = githubCmd.MarkFlagRequired("repo")
-
- githubCmd.Run = func(cmd *cobra.Command, args []string) {
- runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- "owner": *owner,
- "repo": *repo,
- })
- }
- runner.RunCmd(githubCmd)
-}
diff --git a/plugins/github/models/connection.go
b/plugins/github/models/connection.go
index 34303c91..afc7577f 100644
--- a/plugins/github/models/connection.go
+++ b/plugins/github/models/connection.go
@@ -33,6 +33,7 @@ type GithubConnection struct {
type Config struct {
PrType string `mapstructure:"prType" json:"prType"`
PrComponent string `mapstructure:"prComponent"
json:"prComponent"`
+ PrBodyClosePattern string `mapstructure:"prBodyClosePattern"
json:"prBodyClosePattern"`
IssueSeverity string `mapstructure:"issueSeverity"
json:"issueSeverity"`
IssuePriority string `mapstructure:"issuePriority"
json:"issuePriority"`
IssueComponent string `mapstructure:"issueComponent"
json:"issueComponent"`
diff --git a/plugins/github/models/migrationscripts/archived/commit_stat.go
b/plugins/github/models/migrationscripts/archived/commit_stat.go
index 828d4dbc..b9280703 100644
--- a/plugins/github/models/migrationscripts/archived/commit_stat.go
+++ b/plugins/github/models/migrationscripts/archived/commit_stat.go
@@ -24,9 +24,10 @@ import (
)
type GithubCommitStat struct {
- Sha string `gorm:"primaryKey;type:varchar(40)"`
- Additions int `gorm:"comment:Added lines of code"`
- Deletions int `gorm:"comment:Deleted lines of code"`
+ Sha string `gorm:"primaryKey;type:varchar(40)"`
+ Additions int `gorm:"comment:Added lines of code"`
+ Deletions int `gorm:"comment:Deleted lines of code"`
+
CommittedDate time.Time `gorm:"index"`
archived.NoPKModel
}
diff --git a/plugins/github/models/migrationscripts/archived/issue.go
b/plugins/github/models/migrationscripts/archived/issue.go
index 17a49fcb..ff465b8b 100644
--- a/plugins/github/models/migrationscripts/archived/issue.go
+++ b/plugins/github/models/migrationscripts/archived/issue.go
@@ -33,6 +33,8 @@ type GithubIssue struct {
Priority string `gorm:"type:varchar(255)"`
Type string `gorm:"type:varchar(100)"`
Status string `gorm:"type:varchar(255)"`
+ AuthorId int
+ AuthorName string `gorm:"type:varchar(255)"`
AssigneeId int
AssigneeName string `gorm:"type:varchar(255)"`
LeadTimeMinutes uint
diff --git a/plugins/github/models/migrationscripts/archived/pull_request.go
b/plugins/github/models/migrationscripts/archived/pull_request.go
index 43895e7d..ef4ff04b 100644
--- a/plugins/github/models/migrationscripts/archived/pull_request.go
+++ b/plugins/github/models/migrationscripts/archived/pull_request.go
@@ -50,7 +50,7 @@ type GithubPullRequest struct {
HeadCommitSha string `gorm:"type:varchar(255)"`
Url string `gorm:"type:varchar(255)"`
AuthorName string `gorm:"type:varchar(100)"`
- AuthorId string `gorm:"type:varchar(100)"`
+ AuthorId int
archived.NoPKModel
}
diff --git a/plugins/github/models/migrationscripts/archived/repo.go
b/plugins/github/models/migrationscripts/archived/repo.go
index 2a514a2a..7ecee73b 100644
--- a/plugins/github/models/migrationscripts/archived/repo.go
+++ b/plugins/github/models/migrationscripts/archived/repo.go
@@ -24,9 +24,9 @@ import (
)
type GithubRepo struct {
- GithubId int `gorm:"primaryKey"`
- Name string
- HTMLUrl string
+ GithubId int `gorm:"primaryKey"`
+ Name string `gorm:"type:varchar(255)"`
+ HTMLUrl string `gorm:"type:varchar(255)"`
Description string
OwnerId int `json:"ownerId"`
OwnerLogin string `json:"ownerLogin" gorm:"type:varchar(255)"`
diff --git a/plugins/github/models/migrationscripts/init_schema.go
b/plugins/github/models/migrationscripts/init_schema.go
deleted file mode 100644
index b924cb23..00000000
--- a/plugins/github/models/migrationscripts/init_schema.go
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package migrationscripts
-
-import (
- "context"
-
-
"github.com/apache/incubator-devlake/plugins/github/models/migrationscripts/archived"
- "gorm.io/gorm"
-)
-
-type InitSchemas struct{}
-
-func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
- return db.Migrator().AutoMigrate(
- &archived.GithubRepo{},
- &archived.GithubCommit{},
- &archived.GithubRepoCommit{},
- &archived.GithubPullRequest{},
- &archived.GithubReviewer{},
- &archived.GithubPullRequestComment{},
- &archived.GithubPullRequestCommit{},
- &archived.GithubPullRequestLabel{},
- &archived.GithubIssue{},
- &archived.GithubIssueComment{},
- &archived.GithubIssueEvent{},
- &archived.GithubIssueLabel{},
- &archived.GithubUser{},
- &archived.GithubPullRequestIssue{},
- &archived.GithubCommitStat{})
-}
-
-func (*InitSchemas) Version() uint64 {
- return 20220407201135
-}
-
-func (*InitSchemas) Name() string {
- return "Github init schemas"
-}
diff --git a/plugins/github/models/migrationscripts/init_schema_20220610.go
b/plugins/github/models/migrationscripts/init_schema_20220610.go
new file mode 100644
index 00000000..d684f767
--- /dev/null
+++ b/plugins/github/models/migrationscripts/init_schema_20220610.go
@@ -0,0 +1,139 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "context"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ commonArchived
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
+
"github.com/apache/incubator-devlake/plugins/github/models/migrationscripts/archived"
+ "gorm.io/gorm"
+)
+
+type GithubConnection20220609 struct {
+ commonArchived.Model
+ Name string `gorm:"type:varchar(100);uniqueIndex" json:"name"
validate:"required"`
+ Endpoint string `mapstructure:"endpoint" env:"GITHUB_ENDPOINT"
validate:"required"`
+ Proxy string `mapstructure:"proxy" env:"GITHUB_PROXY"`
+ RateLimit int `comment:"api request rate limit per hour"`
+ Token string `mapstructure:"token" env:"GITHUB_AUTH"
validate:"required" encrypt:"yes"`
+}
+
+func (GithubConnection20220609) TableName() string {
+ return "_tool_github_connections"
+}
+
+type InitSchemas struct {
+ config core.ConfigGetter
+}
+
+func (u *InitSchemas) SetConfigGetter(config core.ConfigGetter) {
+ u.config = config
+}
+
+func (u *InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
+ if db.Migrator().HasTable(GithubConnection20220609{}) {
+ err := db.Migrator().DropTable(GithubConnection20220609{})
+ if err != nil {
+ return err
+ }
+ }
+ err := db.Migrator().DropTable(
+ &archived.GithubRepo{},
+ &archived.GithubCommit{},
+ &archived.GithubRepoCommit{},
+ &archived.GithubPullRequest{},
+ &archived.GithubReviewer{},
+ &archived.GithubPullRequestComment{},
+ &archived.GithubPullRequestCommit{},
+ &archived.GithubPullRequestLabel{},
+ &archived.GithubIssue{},
+ &archived.GithubIssueComment{},
+ &archived.GithubIssueEvent{},
+ &archived.GithubIssueLabel{},
+ &archived.GithubUser{},
+ &archived.GithubPullRequestIssue{},
+ &archived.GithubCommitStat{},
+ "_raw_github_api_issues",
+ "_raw_github_api_comments",
+ "_raw_github_api_commits",
+ "_raw_github_api_commit_stats",
+ "_raw_github_api_events",
+ "_raw_github_api_issues",
+ "_raw_github_api_pull_requests",
+ "_raw_github_api_pull_request_commits",
+ "_raw_github_api_pull_request_reviews",
+ "_raw_github_api_repositories",
+ )
+
+ // create connection
+ if err != nil {
+ return err
+ }
+ err = db.Migrator().CreateTable(GithubConnection20220609{})
+ if err != nil {
+ return err
+ }
+ connection := &GithubConnection20220609{}
+ connection.Endpoint = u.config.GetString(`GITHUB_ENDPOINT`)
+ connection.Proxy = u.config.GetString(`GITHUB_PROXY`)
+ connection.Token = u.config.GetString(`GITHUB_AUTH`)
+ connection.Name = `GitHub`
+ if err != nil {
+ return err
+ }
+ err = helper.UpdateEncryptFields(connection, func(plaintext string)
(string, error) {
+ return core.Encrypt(u.config.GetString(core.EncodeKeyEnvStr),
plaintext)
+ })
+ if err != nil {
+ return err
+ }
+ // update from .env and save to db
+ if connection.Endpoint != `` && connection.Token != `` {
+ db.Create(connection)
+ }
+
+ // create other table with connection id
+ return db.Migrator().AutoMigrate(
+ &archived.GithubRepo{},
+ &archived.GithubCommit{},
+ &archived.GithubRepoCommit{},
+ &archived.GithubPullRequest{},
+ &archived.GithubReviewer{},
+ &archived.GithubPullRequestComment{},
+ &archived.GithubPullRequestCommit{},
+ &archived.GithubPullRequestLabel{},
+ &archived.GithubIssue{},
+ &archived.GithubIssueComment{},
+ &archived.GithubIssueEvent{},
+ &archived.GithubIssueLabel{},
+ &archived.GithubUser{},
+ &archived.GithubPullRequestIssue{},
+ &archived.GithubCommitStat{},
+ )
+}
+
+func (*InitSchemas) Version() uint64 {
+ return 20220610000001
+}
+
+func (*InitSchemas) Name() string {
+ return "Github init schemas 20220610"
+}
diff --git a/plugins/github/models/migrationscripts/updateSchemas20220509.go
b/plugins/github/models/migrationscripts/updateSchemas20220509.go
deleted file mode 100644
index c4f76032..00000000
--- a/plugins/github/models/migrationscripts/updateSchemas20220509.go
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package migrationscripts
-
-import (
- "context"
- "github.com/apache/incubator-devlake/models/migrationscripts/archived"
- "time"
-
- "gorm.io/gorm"
-)
-
-type GithubPullRequest20220509 struct {
- GithubId int `gorm:"primaryKey"`
- RepoId int `gorm:"index"`
- Number int `gorm:"index"` // This number is used in GET
requests to the API associated to reviewers / comments / etc.
- State string `gorm:"type:varchar(255)"`
- Title string `gorm:"type:varchar(255)"`
- GithubCreatedAt time.Time
- GithubUpdatedAt time.Time `gorm:"index"`
- ClosedAt *time.Time
- // In order to get the following fields, we need to collect PRs
individually from GitHub
- Additions int
- Deletions int
- Comments int
- Commits int
- ReviewComments int
- Merged bool
- MergedAt *time.Time
- Body string
- Type string `gorm:"type:varchar(255)"`
- Component string `gorm:"type:varchar(255)"`
- MergeCommitSha string `gorm:"type:varchar(40)"`
- HeadRef string `gorm:"type:varchar(255)"`
- BaseRef string `gorm:"type:varchar(255)"`
- BaseCommitSha string `gorm:"type:varchar(255)"`
- HeadCommitSha string `gorm:"type:varchar(255)"`
- Url string `gorm:"type:varchar(255)"`
- AuthorName string `gorm:"type:varchar(100)"`
- AuthorId int
- archived.NoPKModel
-}
-
-func (GithubPullRequest20220509) TableName() string {
- return "_tool_github_pull_requests"
-}
-
-type UpdateSchemas20220509 struct{}
-
-func (*UpdateSchemas20220509) Up(ctx context.Context, db *gorm.DB) error {
- if db.Name() == "postgres" {
- err := db.Exec("ALTER TABLE _tool_github_pull_requests ALTER
COLUMN author_id TYPE bigint USING author_id::bigint").Error
- if err != nil {
- return err
- }
- return nil
- }
- err := db.Migrator().AlterColumn(GithubPullRequest20220509{},
"author_id")
- if err != nil {
- return err
- }
- return nil
-}
-
-func (*UpdateSchemas20220509) Version() uint64 {
- return 20220509212344
-}
-
-func (*UpdateSchemas20220509) Name() string {
- return "Alter pr column `author_id` to int"
-}
diff --git a/plugins/github/models/migrationscripts/updateSchemas20220524.go
b/plugins/github/models/migrationscripts/updateSchemas20220524.go
deleted file mode 100644
index 2196cab2..00000000
--- a/plugins/github/models/migrationscripts/updateSchemas20220524.go
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package migrationscripts
-
-import (
- "context"
- "gorm.io/gorm"
-)
-
-type GithubIssue20220524 struct {
- AuthorId int
- AuthorName string `gorm:"type:varchar(255)"`
-}
-
-func (GithubIssue20220524) TableName() string {
- return "_tool_github_issues"
-}
-
-type UpdateSchemas20220524 struct{}
-
-func (*UpdateSchemas20220524) Up(ctx context.Context, db *gorm.DB) error {
- err := db.Migrator().AddColumn(GithubIssue20220524{}, "author_id")
- if err != nil {
- return err
- }
- err = db.Migrator().AddColumn(GithubIssue20220524{}, "author_name")
- if err != nil {
- return err
- }
- return nil
-}
-
-func (*UpdateSchemas20220524) Version() uint64 {
- return 20220524000002
-}
-
-func (*UpdateSchemas20220524) Name() string {
- return "Add column `author_id`/`author_name` in `GithubIssue`"
-}
diff --git a/plugins/github/models/migrationscripts/updateSchemas20220608.go
b/plugins/github/models/migrationscripts/updateSchemas20220608.go
deleted file mode 100644
index 679579b4..00000000
--- a/plugins/github/models/migrationscripts/updateSchemas20220608.go
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package migrationscripts
-
-import (
- "context"
- "github.com/apache/incubator-devlake/config"
- "github.com/apache/incubator-devlake/models/migrationscripts/archived"
- "github.com/apache/incubator-devlake/plugins/helper"
- "gorm.io/gorm"
-)
-
-type GithubConnection20220608 struct {
- archived.Model
- Name string `gorm:"type:varchar(100);uniqueIndex" json:"name"
validate:"required"`
- Endpoint string `mapstructure:"endpoint" env:"GITHUB_ENDPOINT"
validate:"required"`
- Proxy string `mapstructure:"proxy" env:"GITHUB_PROXY"`
- RateLimit int `comment:"api request rate limit per hour"`
- Token string `mapstructure:"token" validate:"required"
env:"GITHUB_AUTH"`
-
- Config20220608 `mapstructure:",squash"`
-}
-
-type Config20220608 struct {
- PrType string `mapstructure:"prType" env:"GITHUB_PR_TYPE"
json:"prType"`
- PrComponent string `mapstructure:"prComponent"
env:"GITHUB_PR_COMPONENT" json:"prComponent"`
- IssueSeverity string `mapstructure:"issueSeverity"
env:"GITHUB_ISSUE_SEVERITY" json:"issueSeverity"`
- IssuePriority string `mapstructure:"issuePriority"
env:"GITHUB_ISSUE_PRIORITY" json:"issuePriority"`
- IssueComponent string `mapstructure:"issueComponent"
env:"GITHUB_ISSUE_COMPONENT" json:"issueComponent"`
- IssueTypeBug string `mapstructure:"issueTypeBug"
env:"GITHUB_ISSUE_TYPE_BUG" json:"issueTypeBug"`
- IssueTypeIncident string `mapstructure:"issueTypeIncident"
env:"GITHUB_ISSUE_TYPE_INCIDENT" json:"issueTypeIncident"`
- IssueTypeRequirement string `mapstructure:"issueTypeRequirement"
env:"GITHUB_ISSUE_TYPE_REQUIREMENT" json:"issueTypeRequirement"`
-}
-
-func (GithubConnection20220608) TableName() string {
- return "_tool_github_connections"
-}
-
-type UpdateSchemas20220608 struct{}
-
-func (*UpdateSchemas20220608) Up(ctx context.Context, db *gorm.DB) error {
- err := db.Migrator().CreateTable(GithubConnection20220608{})
- if err != nil {
- return err
- }
- v := config.GetConfig()
- connection := &GithubConnection20220608{}
- err = helper.EncodeStruct(v, connection, "env")
- connection.Name = `GitHub`
- if err != nil {
- return err
- }
- // update from .env and save to db
- if connection.Endpoint != `` && connection.Token != `` {
- db.Create(connection)
- }
- return nil
-}
-
-func (*UpdateSchemas20220608) Version() uint64 {
- return 20220608000003
-}
-
-func (*UpdateSchemas20220608) Name() string {
- return "Add connection for github"
-}
diff --git a/plugins/github/tasks/comment_collector.go
b/plugins/github/tasks/comment_collector.go
index e7c22989..9263a312 100644
--- a/plugins/github/tasks/comment_collector.go
+++ b/plugins/github/tasks/comment_collector.go
@@ -31,7 +31,7 @@ import (
const RAW_COMMENTS_TABLE = "github_api_comments"
-// this struct should be moved to `gitub_api_common.go`
+// this struct should be moved to `github_api_common.go`
func CollectApiComments(taskCtx core.SubTaskContext) error {
db := taskCtx.GetDb()
@@ -72,7 +72,6 @@ func CollectApiComments(taskCtx core.SubTaskContext) error {
since = &latestUpdatedPrComt.GithubUpdatedAt
incremental = true
}
-
}
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
diff --git a/plugins/github/tasks/pr_issue_enricher.go
b/plugins/github/tasks/pr_issue_enricher.go
index 6fa108dc..f16beacc 100644
--- a/plugins/github/tasks/pr_issue_enricher.go
+++ b/plugins/github/tasks/pr_issue_enricher.go
@@ -40,7 +40,7 @@ func EnrichPullRequestIssues(taskCtx core.SubTaskContext)
(err error) {
repoId := data.Repo.GithubId
var prBodyCloseRegex *regexp.Regexp
- prBodyClosePattern := taskCtx.GetConfig("GITHUB_PR_BODY_CLOSE_PATTERN")
+ prBodyClosePattern := data.Options.PrBodyClosePattern
//the pattern before the issue number, sometimes, the issue number is
#1098, sometimes it is https://xxx/#1098
prBodyClosePattern = strings.Replace(prBodyClosePattern, "%s",
data.Options.Owner, 1)
prBodyClosePattern = strings.Replace(prBodyClosePattern, "%s",
data.Options.Repo, 1)
diff --git a/plugins/github/tasks/task_data.go
b/plugins/github/tasks/task_data.go
index f48c77bf..eb551dc1 100644
--- a/plugins/github/tasks/task_data.go
+++ b/plugins/github/tasks/task_data.go
@@ -25,11 +25,12 @@ import (
)
type GithubOptions struct {
- Tasks []string `json:"tasks,omitempty"`
- Since string
- Owner string
- Repo string
- models.Config
+ ConnectionId uint64 `json:"connectionId"`
+ Tasks []string `json:"tasks,omitempty"`
+ Since string
+ Owner string
+ Repo string
+ models.Config `mapstructure:",squash"`
}
type GithubTaskData struct {
