This is an automated email from the ASF dual-hosted git repository.
mappjzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new f103c712 Commit file (#3062)
f103c712 is described below
commit f103c71218e772df39f17f25acb0660b18916e4e
Author: mappjzc <[email protected]>
AuthorDate: Thu Sep 15 16:50:19 2022 +0800
Commit file (#3062)
* fix: add length of commit_file
changed file_path from 255 to text
changed id fromcommitsha and path to commtsha and sha path
Nddtfjiang <[email protected]>
* fix: fix migrationscripts use domainlayer or common
changed domainlayer and common to be archived.
Nddtfjiang <[email protected]>
* fix: use batch save for update table
Use batch_save for update table.
Nddtfjiang <[email protected]>
* style: add jira source struct for review
Add JiraSource for review.
Nddtfjiang <[email protected]>
* fix: fix commit_file_components update lost
fix commit_file_components update lost.
Nddtfjiang <[email protected]>
Co-authored-by: Warren Chen <[email protected]>
---
models/domainlayer/code/commit.go | 5 +-
.../20220601_add_subtasks_field.go | 4 +-
.../20220722_commitfile_component.go | 7 +-
.../20220801_add_NoPKModel_to_CommitParent.go | 5 +-
.../migrationscripts/20220904_encrypt_pipeline.go | 6 +-
.../20220913_commitfile_add_length.go | 195 +++++++++++++++++++++
models/migrationscripts/register.go | 1 +
.../archived/meeting_top_user_item.go | 17 +-
.../models/migrationscripts/archived/commit.go | 4 +-
plugins/gitextractor/parser/repo.go | 16 +-
plugins/helper/batch_save.go | 3 +-
.../20220505_rename_source_table.go | 24 ++-
.../migrationscripts/archived/iteration_story.go | 4 +-
13 files changed, 259 insertions(+), 32 deletions(-)
diff --git a/models/domainlayer/code/commit.go
b/models/domainlayer/code/commit.go
index 73145539..77f2d61c 100644
--- a/models/domainlayer/code/commit.go
+++ b/models/domainlayer/code/commit.go
@@ -18,9 +18,10 @@ limitations under the License.
package code
import (
- "github.com/apache/incubator-devlake/models/domainlayer"
"time"
+ "github.com/apache/incubator-devlake/models/domainlayer"
+
"github.com/apache/incubator-devlake/models/common"
)
@@ -48,7 +49,7 @@ func (Commit) TableName() string {
type CommitFile struct {
domainlayer.DomainEntity
CommitSha string `gorm:"type:varchar(40)"`
- FilePath string `gorm:"type:varchar(255)"`
+ FilePath string `gorm:"type:text"`
Additions int
Deletions int
}
diff --git a/models/migrationscripts/20220601_add_subtasks_field.go
b/models/migrationscripts/20220601_add_subtasks_field.go
index 6ee631f7..e49ee2b7 100644
--- a/models/migrationscripts/20220601_add_subtasks_field.go
+++ b/models/migrationscripts/20220601_add_subtasks_field.go
@@ -22,13 +22,13 @@ import (
"github.com/apache/incubator-devlake/errors"
"time"
- "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
"gorm.io/datatypes"
"gorm.io/gorm"
)
type Task20220601 struct {
- common.Model
+ archived.Model
Plugin string `json:"plugin" gorm:"index"`
Subtasks datatypes.JSON `json:"subtasks"`
Options datatypes.JSON `json:"options"`
diff --git a/models/migrationscripts/20220722_commitfile_component.go
b/models/migrationscripts/20220722_commitfile_component.go
index 8102f1f9..544c8014 100644
--- a/models/migrationscripts/20220722_commitfile_component.go
+++ b/models/migrationscripts/20220722_commitfile_component.go
@@ -19,9 +19,8 @@ package migrationscripts
import (
"context"
+
"github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/models/common"
- "github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
"gorm.io/gorm"
)
@@ -37,7 +36,7 @@ func (Component) TableName() string {
}
type CommitFile struct {
- domainlayer.DomainEntity
+ archived.DomainEntity
CommitSha string `gorm:"type:varchar(40)"`
FilePath string `gorm:"type:varchar(255)"`
Additions int
@@ -49,7 +48,7 @@ func (CommitFile) TableName() string {
}
type CommitFileComponent struct {
- common.NoPKModel
+ archived.NoPKModel
CommitFileId string `gorm:"primaryKey;type:varchar(255)"`
ComponentName string `gorm:"type:varchar(255)"`
}
diff --git a/models/migrationscripts/20220801_add_NoPKModel_to_CommitParent.go
b/models/migrationscripts/20220801_add_NoPKModel_to_CommitParent.go
index e7269820..760b0648 100644
--- a/models/migrationscripts/20220801_add_NoPKModel_to_CommitParent.go
+++ b/models/migrationscripts/20220801_add_NoPKModel_to_CommitParent.go
@@ -19,13 +19,14 @@ package migrationscripts
import (
"context"
+
"github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
"gorm.io/gorm"
)
type commitParent struct {
- common.NoPKModel
+ archived.NoPKModel
CommitSha string `json:"commitSha"
gorm:"primaryKey;type:varchar(40);comment:commit hash"`
ParentCommitSha string `json:"parentCommitSha"
gorm:"primaryKey;type:varchar(40);comment:parent commit hash"`
}
diff --git a/models/migrationscripts/20220904_encrypt_pipeline.go
b/models/migrationscripts/20220904_encrypt_pipeline.go
index 63c35297..b2e9e26e 100644
--- a/models/migrationscripts/20220904_encrypt_pipeline.go
+++ b/models/migrationscripts/20220904_encrypt_pipeline.go
@@ -19,13 +19,15 @@ package migrationscripts
import (
"context"
+ "time"
+
"github.com/apache/incubator-devlake/config"
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
"github.com/apache/incubator-devlake/plugins/core"
"gorm.io/datatypes"
"gorm.io/gorm"
- "time"
)
type Pipeline0904Temp struct {
@@ -48,7 +50,7 @@ func (Pipeline0904Temp) TableName() string {
}
type PipelineOldVersion struct {
- common.Model
+ archived.Model
Name string `json:"name" gorm:"index"`
BlueprintId uint64 `json:"blueprintId"`
Plan datatypes.JSON `json:"plan"`
diff --git a/models/migrationscripts/20220913_commitfile_add_length.go
b/models/migrationscripts/20220913_commitfile_add_length.go
new file mode 100644
index 00000000..9b5a5cb0
--- /dev/null
+++ b/models/migrationscripts/20220913_commitfile_add_length.go
@@ -0,0 +1,195 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "reflect"
+ "strings"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/plugins/gitlab/api"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "gorm.io/gorm"
+)
+
+type CommitFileAddLength struct {
+ archived.DomainEntity
+ CommitSha string `gorm:"type:varchar(40)"`
+ FilePath string `gorm:"type:text"`
+ Additions int
+ Deletions int
+}
+
+func (CommitFileAddLength) TableName() string {
+ return "commit_files"
+}
+
+type CommitFileAddLengthBak struct {
+ archived.DomainEntity
+ CommitSha string `gorm:"type:varchar(40)"`
+ FilePath string `gorm:"type:varchar(255)"`
+ Additions int
+ Deletions int
+}
+
+func (CommitFileAddLengthBak) TableName() string {
+ return "commit_files_bak"
+}
+
+type CommitFileComponentBak struct {
+ archived.NoPKModel
+ CommitFileId string `gorm:"primaryKey;type:varchar(255)"`
+ ComponentName string `gorm:"type:varchar(255)"`
+}
+
+func (CommitFileComponentBak) TableName() string {
+ return "commit_file_components_bak"
+}
+
+type addCommitFilePathLength struct{}
+
+func (*addCommitFilePathLength) Up(ctx context.Context, db *gorm.DB)
errors.Error {
+ err := db.Migrator().RenameTable(&CommitFile{},
&CommitFileAddLengthBak{})
+ if err != nil {
+ return errors.Default.Wrap(err, "error no rename commit_file to
commit_files_bak")
+ }
+
+ err = db.Migrator().AutoMigrate(&CommitFileAddLength{})
+ if err != nil {
+ return errors.Default.Wrap(err, "error on auto migrate
commit_file")
+ }
+
+ // update old id to new id and write to the new table
+ cursor, err := db.Model(&CommitFileAddLengthBak{}).Rows()
+ if err != nil {
+ return errors.Default.Wrap(err, "error on select
CommitFileAddLength")
+ }
+ defer cursor.Close()
+
+ batch, err := helper.NewBatchSave(api.BasicRes,
reflect.TypeOf(&CommitFileAddLength{}), 200)
+ if err != nil {
+ return errors.Default.Wrap(err, "error getting batch from table
commit_file")
+ }
+
+ defer batch.Close()
+ for cursor.Next() {
+ cfb := CommitFileAddLengthBak{}
+ err = db.ScanRows(cursor, &cfb)
+ if err != nil {
+ return errors.Default.Wrap(err, "error scan rows from
table commit_files_bak")
+ }
+
+ cf := CommitFileAddLength(cfb)
+
+ // With some long path,the varchar(255) was not enough both ID
and file_path
+ // So we use the hash to compress the path in ID and add length
of file_path.
+ shaFilePath := sha256.New()
+ shaFilePath.Write([]byte(cf.FilePath))
+ cf.Id = cf.CommitSha + ":" +
hex.EncodeToString(shaFilePath.Sum(nil))
+
+ err = batch.Add(&cf)
+ if err != nil {
+ return errors.Default.Wrap(err, "error on batch add")
+ }
+ }
+
+ err = db.Migrator().RenameTable(&CommitFileComponent{},
&CommitFileComponentBak{})
+ if err != nil {
+ return errors.Default.Wrap(err, "error no rename
commit_file_components to commit_file_components_bak")
+ }
+
+ err = db.Migrator().AutoMigrate(&CommitFileComponent{})
+ if err != nil {
+ return errors.Default.Wrap(err, "error on auto migrate
commit_file")
+ }
+
+ // update old id to new id and write to the new table
+ cursor2, err := db.Model(&CommitFileComponentBak{}).Rows()
+ if err != nil {
+ return errors.Default.Wrap(err, "error on select
commit_file_components_bak")
+ }
+ defer cursor2.Close()
+
+ batch2, err := helper.NewBatchSave(api.BasicRes,
reflect.TypeOf(&CommitFileComponent{}), 500)
+ if err != nil {
+ return errors.Default.Wrap(err, "error getting batch from table
commit_file_components")
+ }
+ defer batch2.Close()
+
+ for cursor2.Next() {
+ cfcb := CommitFileComponentBak{}
+ err = db.ScanRows(cursor2, &cfcb)
+ if err != nil {
+ return errors.Default.Wrap(err, "error scan rows from
table commit_file_components_bak")
+ }
+
+ cfc := CommitFileComponent(cfcb)
+
+ ids := strings.Split(cfc.CommitFileId, ":")
+
+ commitSha := ""
+ filePath := ""
+
+ if len(ids) > 0 {
+ commitSha = ids[0]
+ if len(ids) > 1 {
+ for i := 1; i < len(ids); i++ {
+ if i > 1 {
+ filePath += ":"
+ }
+ filePath += ids[i]
+ }
+ }
+ }
+
+ // With some long path,the varchar(255) was not enough both ID
and file_path
+ // So we use the hash to compress the path in ID and add length
of file_path.
+ shaFilePath := sha256.New()
+ shaFilePath.Write([]byte(filePath))
+ cfc.CommitFileId = commitSha + ":" +
hex.EncodeToString(shaFilePath.Sum(nil))
+
+ err = batch2.Add(&cfc)
+ if err != nil {
+ return errors.Default.Wrap(err, "error on batch add")
+ }
+ }
+
+ // drop the old table
+ err = db.Migrator().DropTable(&CommitFileAddLengthBak{})
+ if err != nil {
+ return errors.Default.Wrap(err, "error no drop
commit_files_bak")
+ }
+ err = db.Migrator().DropTable(&CommitFileComponentBak{})
+ if err != nil {
+ return errors.Default.Wrap(err, "error no drop
commit_files_bak")
+ }
+
+ return nil
+}
+
+func (*addCommitFilePathLength) Version() uint64 {
+ return 20220913165805
+}
+
+func (*addCommitFilePathLength) Name() string {
+ return "add length of commit_file file_path"
+}
diff --git a/models/migrationscripts/register.go
b/models/migrationscripts/register.go
index 77c8e71d..d90dffc5 100644
--- a/models/migrationscripts/register.go
+++ b/models/migrationscripts/register.go
@@ -45,5 +45,6 @@ func All() []migration.Script {
new(modifyCICDTasks),
new(modifyBoardRepos),
new(addOriginChangeValueForPr),
+ new(addCommitFilePathLength),
}
}
diff --git
a/plugins/feishu/models/migrationscripts/archived/meeting_top_user_item.go
b/plugins/feishu/models/migrationscripts/archived/meeting_top_user_item.go
index 733f9897..e315caae 100644
--- a/plugins/feishu/models/migrationscripts/archived/meeting_top_user_item.go
+++ b/plugins/feishu/models/migrationscripts/archived/meeting_top_user_item.go
@@ -18,18 +18,19 @@ limitations under the License.
package archived
import (
- "github.com/apache/incubator-devlake/models/common"
"time"
+
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
)
type FeishuMeetingTopUserItem struct {
- common.NoPKModel `json:"-"`
- ConnectionId uint64 `gorm:"primaryKey"`
- StartTime time.Time `gorm:"primaryKey"`
- Name string `json:"name"
gorm:"primaryKey;type:varchar(255)"`
- MeetingCount string `json:"meeting_count"
gorm:"type:varchar(255)"`
- MeetingDuration string `json:"meeting_duration"
gorm:"type:varchar(255)"`
- UserType int64 `json:"user_type"`
+ archived.NoPKModel `json:"-"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ StartTime time.Time `gorm:"primaryKey"`
+ Name string `json:"name"
gorm:"primaryKey;type:varchar(255)"`
+ MeetingCount string `json:"meeting_count"
gorm:"type:varchar(255)"`
+ MeetingDuration string `json:"meeting_duration"
gorm:"type:varchar(255)"`
+ UserType int64 `json:"user_type"`
}
func (FeishuMeetingTopUserItem) TableName() string {
diff --git a/plugins/gitee/models/migrationscripts/archived/commit.go
b/plugins/gitee/models/migrationscripts/archived/commit.go
index 82c2c570..e170e7ff 100644
--- a/plugins/gitee/models/migrationscripts/archived/commit.go
+++ b/plugins/gitee/models/migrationscripts/archived/commit.go
@@ -20,7 +20,7 @@ package archived
import (
"time"
- "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
)
type GiteeCommit struct {
@@ -39,7 +39,7 @@ type GiteeCommit struct {
Additions int `gorm:"comment:Added lines of code"`
Deletions int `gorm:"comment:Deleted lines of code"`
Total int `gorm:"comment:Sum of added/deleted lines of
code"`
- common.NoPKModel
+ archived.NoPKModel
}
func (GiteeCommit) TableName() string {
diff --git a/plugins/gitextractor/parser/repo.go
b/plugins/gitextractor/parser/repo.go
index 9795468a..caa73177 100644
--- a/plugins/gitextractor/parser/repo.go
+++ b/plugins/gitextractor/parser/repo.go
@@ -19,7 +19,11 @@ package parser
import (
"context"
+ "crypto/sha256"
+ "encoding/hex"
"fmt"
+ "regexp"
+
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/code"
@@ -27,7 +31,6 @@ import (
"github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/gitextractor/models"
git "github.com/libgit2/git2go/v33"
- "regexp"
)
type GitRepo struct {
@@ -329,7 +332,14 @@ func (r *GitRepo) storeCommitFilesFromDiff(commitSha
string, diff *git.Diff, com
commitFile = new(code.CommitFile)
commitFile.CommitSha = commitSha
commitFile.FilePath = file.NewFile.Path
- commitFile.Id = commitSha + ":" + file.NewFile.Path
+
+ // With some long path,the varchar(255) was not enough both ID
and file_path
+ // So we use the hash to compress the path in ID and add length
of file_path.
+ // Use commitSha and the sha256 of FilePath to create id
+ shaFilePath := sha256.New()
+ shaFilePath.Write([]byte(file.NewFile.Path))
+ commitFile.Id = commitSha + ":" +
hex.EncodeToString(shaFilePath.Sum(nil))
+
commitFileComponent = new(code.CommitFileComponent)
for component, reg := range componentMap {
if reg.MatchString(commitFile.FilePath) {
@@ -337,7 +347,7 @@ func (r *GitRepo) storeCommitFilesFromDiff(commitSha
string, diff *git.Diff, com
break
}
}
- commitFileComponent.CommitFileId = commitSha + ":" +
file.NewFile.Path
+ commitFileComponent.CommitFileId = commitFile.Id
//commitFileComponent.FilePath = file.NewFile.Path
//commitFileComponent.CommitSha = commitSha
if commitFileComponent.ComponentName == "" {
diff --git a/plugins/helper/batch_save.go b/plugins/helper/batch_save.go
index bfd8b4d1..f41df20b 100644
--- a/plugins/helper/batch_save.go
+++ b/plugins/helper/batch_save.go
@@ -19,10 +19,11 @@ package helper
import (
"fmt"
- "github.com/apache/incubator-devlake/errors"
"reflect"
"strings"
+ "github.com/apache/incubator-devlake/errors"
+
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/core/dal"
)
diff --git
a/plugins/jira/models/migrationscripts/20220505_rename_source_table.go
b/plugins/jira/models/migrationscripts/20220505_rename_source_table.go
index d95f20d3..56947a15 100644
--- a/plugins/jira/models/migrationscripts/20220505_rename_source_table.go
+++ b/plugins/jira/models/migrationscripts/20220505_rename_source_table.go
@@ -19,14 +19,14 @@ package migrationscripts
import (
"context"
+
"github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/models/common"
-
"github.com/apache/incubator-devlake/plugins/jira/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
"gorm.io/gorm"
)
type JiraConnectionV010 struct {
- common.Model
+ archived.Model
Name string `gorm:"type:varchar(100);uniqueIndex"
json:"name" validate:"required"`
Endpoint string `json:"endpoint" validate:"required"`
BasicAuthEncoded string `json:"basicAuthEncoded"
validate:"required"`
@@ -41,10 +41,26 @@ func (JiraConnectionV010) TableName() string {
return "_tool_jira_connections"
}
+type JiraSource struct {
+ archived.Model
+ Name string `gorm:"type:varchar(100);uniqueIndex"
json:"name" validate:"required"`
+ Endpoint string `json:"endpoint" validate:"required"`
+ BasicAuthEncoded string `json:"basicAuthEncoded"
validate:"required"`
+ EpicKeyField string `gorm:"type:varchar(50);"
json:"epicKeyField"`
+ StoryPointField string `gorm:"type:varchar(50);"
json:"storyPointField"`
+ RemotelinkCommitShaPattern string
`gorm:"type:varchar(255);comment='golang regexp, the first group will be
recognized as commit sha, ref https://github.com/google/re2/wiki/Syntax'"
json:"remotelinkCommitShaPattern"`
+ Proxy string `json:"proxy"`
+ RateLimit int `comment:"api request rate limt per
second"`
+}
+
+func (JiraSource) TableName() string {
+ return "_tool_jira_sources"
+}
+
type renameSourceTable struct{}
func (*renameSourceTable) Up(ctx context.Context, db *gorm.DB) errors.Error {
- err := db.Migrator().RenameTable(archived.JiraSource{},
JiraConnectionV010{})
+ err := db.Migrator().RenameTable(JiraSource{}, JiraConnectionV010{})
return errors.Convert(err)
}
diff --git a/plugins/tapd/models/migrationscripts/archived/iteration_story.go
b/plugins/tapd/models/migrationscripts/archived/iteration_story.go
index c5375191..cbc76d76 100644
--- a/plugins/tapd/models/migrationscripts/archived/iteration_story.go
+++ b/plugins/tapd/models/migrationscripts/archived/iteration_story.go
@@ -18,12 +18,12 @@ limitations under the License.
package archived
import (
- "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
"github.com/apache/incubator-devlake/plugins/helper"
)
type TapdIterationStory struct {
- common.NoPKModel
+ archived.NoPKModel
ConnectionId uint64 `gorm:"primaryKey"`
IterationId uint64 `gorm:"primaryKey"`
WorkspaceId uint64 `gorm:"primaryKey"`
