This is an automated email from the ASF dual-hosted git repository. warren pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit f88e5c22fb6e6699930967ab9c145cab162ec752 Author: Yingchu Chen <[email protected]> AuthorDate: Wed May 25 17:24:19 2022 +0800 feat(api): change jira basicauthencoded to username/password and extract common utils closes #2003 --- api/shared/api_output.go | 2 +- config-ui/src/hooks/useConnectionManager.jsx | 2 +- config-ui/src/pages/configure/settings/jira.jsx | 26 +-- plugins/feishu/tasks/api_client.go | 2 +- plugins/github/tasks/api_client.go | 2 +- plugins/gitlab/tasks/api_client.go | 2 +- plugins/helper/connection.go | 217 +++++++++++++++++++++ plugins/jira/api/connection.go | 191 +++--------------- plugins/jira/api/issue_status_mapping.go | 216 -------------------- plugins/jira/api/issue_type_mapping.go | 214 -------------------- plugins/jira/api/proxy.go | 16 +- plugins/jira/jira.go | 17 +- plugins/jira/models/connection.go | 16 +- .../migrationscripts/updateSchemas20220505.go | 20 +- .../migrationscripts/updateSchemas20220524.go | 72 +++++++ plugins/jira/tasks/api_client.go | 8 +- plugins/tapd/tasks/api_client.go | 4 +- 17 files changed, 353 insertions(+), 674 deletions(-) diff --git a/api/shared/api_output.go b/api/shared/api_output.go index 8e006acd..28e2e630 100644 --- a/api/shared/api_output.go +++ b/api/shared/api_output.go @@ -35,7 +35,7 @@ func ApiOutputError(c *gin.Context, err error, status int) { Message: err.Error(), }) } else { - logger.Global.Error("Server Internal Error: %w", err) + logger.Global.Error("Server Internal Error: %s", err.Error()) c.JSON(status, &ApiBody{ Success: false, Message: err.Error(), diff --git a/config-ui/src/hooks/useConnectionManager.jsx b/config-ui/src/hooks/useConnectionManager.jsx index a3558df0..f9116919 100644 --- a/config-ui/src/hooks/useConnectionManager.jsx +++ b/config-ui/src/hooks/useConnectionManager.jsx @@ -83,7 +83,7 @@ function useConnectionManager ({ let connectionPayload switch (activeProvider.id) { case Providers.JIRA: - connectionPayload = { endpoint: endpointUrl, auth: token, proxy: proxy } + connectionPayload = { endpoint: endpointUrl, username: username, password: password, proxy: proxy } break case Providers.GITHUB: connectionPayload = { endpoint: endpointUrl, auth: token, proxy: proxy } diff --git a/config-ui/src/pages/configure/settings/jira.jsx b/config-ui/src/pages/configure/settings/jira.jsx index 8305eaa0..cda3defc 100644 --- a/config-ui/src/pages/configure/settings/jira.jsx +++ b/config-ui/src/pages/configure/settings/jira.jsx @@ -88,25 +88,6 @@ export default function JiraSettings (props) { : null } - const parseTypeMappings = useCallback((mappings = []) => { - const GroupedMappings = { - [MAPPING_TYPES.Requirement]: [], - [MAPPING_TYPES.Incident]: [], - [MAPPING_TYPES.Bug]: [], - } - Object.entries(mappings).forEach(([tag, typeObj]) => { - GroupedMappings[typeObj.standardType].push(tag) - }) - console.log('>>>> PARSED TYPE MAPPINGS ....', GroupedMappings) - setTypeMappingRequirement(GroupedMappings[MAPPING_TYPES.Requirement]) - setTypeMappingBug(GroupedMappings[MAPPING_TYPES.Bug]) - setTypeMappingIncident(GroupedMappings[MAPPING_TYPES.Incident]) - setRequirementTags(requirementTagsList?.filter(t => GroupedMappings[MAPPING_TYPES.Requirement].includes(t.value))) - setBugTags(bugTagsList?.filter(t => GroupedMappings[MAPPING_TYPES.Bug].includes(t.value))) - setIncidentTags(incidentTagsList?.filter(t => GroupedMappings[MAPPING_TYPES.Incident].includes(t.value))) - return GroupedMappings - }, [requirementTagsList, bugTagsList, incidentTagsList]) - useEffect(() => { const settings = { epicKeyField: jiraIssueEpicKeyField?.value || '', @@ -164,13 +145,12 @@ export default function JiraSettings (props) { console.log('>> CONN SETTINGS OBJECT ', connection) if (connection && connection.ID) { // Parse Type Mappings (V2) - parseTypeMappings(connection.typeMappings) setStatusMappings([]) setRemoteLinkCommitSha(connection.remotelinkCommitShaPattern) // setJiraIssueEpicKeyField(fieldsList.find(f => f.value === connection.epicKeyField)) // setJiraIssueStoryPointField(fieldsList.find(f => f.value === connection.storyPointField)) } - }, [connection, parseTypeMappings]) + }, [connection]) useEffect(() => { setTypeMappingRequirement(requirementTags) @@ -208,10 +188,6 @@ export default function JiraSettings (props) { setJiraIssueStoryPointField(fieldsList.find(f => f.value === connection.storyPointField)) }, [fieldsList, connection.epicKeyField, connection.storyPointField]) - useEffect(() => { - parseTypeMappings(connection.typeMappings) - }, [requirementTagsList, bugTagsList, incidentTagsList, connection.typeMappings, parseTypeMappings]) - return ( <> <div className='headlineContainer'> diff --git a/plugins/feishu/tasks/api_client.go b/plugins/feishu/tasks/api_client.go index d6b7b7d0..da54b5f9 100644 --- a/plugins/feishu/tasks/api_client.go +++ b/plugins/feishu/tasks/api_client.go @@ -92,7 +92,7 @@ func NewFeishuApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error apiClient.SetAfterFunction(func(res *http.Response) error { if res.StatusCode == http.StatusUnauthorized { - return fmt.Errorf("feishu authentication failed, please check your Bearer Auth Token") + return fmt.Errorf("feishu authentication failed, please check your AccessToken") } return nil }) diff --git a/plugins/github/tasks/api_client.go b/plugins/github/tasks/api_client.go index 94a60e05..aa0e9aa0 100644 --- a/plugins/github/tasks/api_client.go +++ b/plugins/github/tasks/api_client.go @@ -60,7 +60,7 @@ func CreateApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error) { }) apiClient.SetAfterFunction(func(res *http.Response) error { if res.StatusCode == http.StatusUnauthorized { - return fmt.Errorf("authentication failed, please check your Token configuration") + return fmt.Errorf("authentication failed, please check your AccessToken configuration") } return nil }) diff --git a/plugins/gitlab/tasks/api_client.go b/plugins/gitlab/tasks/api_client.go index 0dc8bf64..f4c2b653 100644 --- a/plugins/gitlab/tasks/api_client.go +++ b/plugins/gitlab/tasks/api_client.go @@ -54,7 +54,7 @@ func NewGitlabApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error } apiClient.SetAfterFunction(func(res *http.Response) error { if res.StatusCode == http.StatusUnauthorized { - return fmt.Errorf("authentication failed, please check your Basic Auth Token") + return fmt.Errorf("authentication failed, please check your AccessToken") } return nil }) diff --git a/plugins/helper/connection.go b/plugins/helper/connection.go new file mode 100644 index 00000000..93974f73 --- /dev/null +++ b/plugins/helper/connection.go @@ -0,0 +1,217 @@ +package helper + +import ( + "encoding/base64" + "fmt" + "github.com/apache/incubator-devlake/config" + "github.com/apache/incubator-devlake/models/common" + "github.com/apache/incubator-devlake/plugins/core" + "github.com/go-playground/validator/v10" + "github.com/mitchellh/mapstructure" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "reflect" + "strconv" +) + +type BaseConnection struct { + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"` + common.Model +} + +type BasicAuth struct { + Username string `mapstructure:"username" validate:"required" json:"username"` + Password string `mapstructure:"password" validate:"required" json:"password" encrypt:"yes"` +} + +func (ba BasicAuth) GetEncodedToken() string { + return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%v:%v", ba.Username, ba.Password))) +} + +type AccessToken struct { + Token string `mapstructure:"token" validate:"required" json:"token" encrypt:"yes"` +} + +type RestConnection struct { + BaseConnection `mapstructure:",squash"` + Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"` + Proxy string `mapstructure:"proxy" json:"proxy"` + RateLimit int `comment:"api request rate limt per hour" json:"rateLimit"` +} + +// RefreshAndSaveConnection populate from request input into connection which come from REST functions to connection struct and save to DB +// and only change value which `data` has +// mergeFieldsToConnection merges fields from data +// `connection` is the pointer of a plugin connection +// `data` is http request input param +func RefreshAndSaveConnection(connection interface{}, data map[string]interface{}, db *gorm.DB) error { + var err error + // update fields from request body + err = mergeFieldsToConnection(connection, data) + if err != nil { + return err + } + + err = saveToDb(connection, db) + + if err != nil { + return err + } + return nil +} + +func saveToDb(connection interface{}, db *gorm.DB) error { + dataVal := reflect.ValueOf(connection) + if dataVal.Kind() != reflect.Ptr { + panic("entityPtr is not a pointer") + } + + dataType := reflect.Indirect(dataVal).Type() + fieldName := getEncryptField(dataType, "encrypt") + plainPwd := "" + err := doEncrypt(dataVal, fieldName) + if err != nil { + return err + } + err = db.Clauses(clause.OnConflict{UpdateAll: true}).Save(connection).Error + if err != nil { + return err + } + + err = doDecrypt(dataVal, fieldName) + if err != nil { + return err + } + dataVal.Elem().FieldByName(fieldName).Set(reflect.ValueOf(plainPwd)) + + return err +} + +// mergeFieldsToConnection will populate all value in map to connection struct and validate the struct +func mergeFieldsToConnection(specificConnection interface{}, connections ...map[string]interface{}) error { + // decode + for _, connection := range connections { + err := mapstructure.Decode(connection, specificConnection) + if err != nil { + return err + } + } + // validate + vld := validator.New() + err := vld.Struct(specificConnection) + if err != nil { + return err + } + + return nil +} + +func getEncKey() (string, error) { + // encrypt + v := config.GetConfig() + encKey := v.GetString(core.EncodeKeyEnvStr) + if encKey == "" { + // Randomly generate a bunch of encryption keys and set them to config + encKey = core.RandomEncKey() + v.Set(core.EncodeKeyEnvStr, encKey) + err := config.WriteConfig(v) + if err != nil { + return encKey, err + } + } + return encKey, nil +} + +// FindConnectionByInput finds connection from db by parsing request input and decrypt it +func FindConnectionByInput(input *core.ApiResourceInput, connection interface{}, db *gorm.DB) error { + dataVal := reflect.ValueOf(connection) + if dataVal.Kind() != reflect.Ptr { + return fmt.Errorf("connection is not a pointer") + } + + id, err := GetConnectionIdByInputParam(input) + if err != nil { + return fmt.Errorf("invalid connectionId") + } + + err = db.First(connection, id).Error + if err != nil { + fmt.Printf("--- %s", err.Error()) + return err + } + + dataType := reflect.Indirect(dataVal).Type() + + fieldName := getEncryptField(dataType, "encrypt") + return doDecrypt(dataVal, fieldName) + +} + +// GetConnectionIdByInputParam gets connectionId by parsing request input +func GetConnectionIdByInputParam(input *core.ApiResourceInput) (uint64, error) { + connectionId := input.Params["connectionId"] + if connectionId == "" { + return 0, fmt.Errorf("missing connectionId") + } + return strconv.ParseUint(connectionId, 10, 64) +} + +func getEncryptField(t reflect.Type, tag string) string { + fieldName := "" + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + if field.Type.Kind() == reflect.Struct { + fieldName = getEncryptField(field.Type, tag) + } else { + if field.Tag.Get(tag) == "yes" { + fieldName = field.Name + } + } + } + return fieldName +} + +// DecryptConnection decrypts password/token field for connection +func DecryptConnection(connection interface{}, fieldName string) error { + dataVal := reflect.ValueOf(connection) + if dataVal.Kind() != reflect.Ptr { + panic("connection is not a pointer") + } + if len(fieldName) == 0 { + dataType := reflect.Indirect(dataVal).Type() + fieldName = getEncryptField(dataType, "encrypt") + } + return doDecrypt(dataVal, fieldName) +} + +func doDecrypt(dataVal reflect.Value, fieldName string) error { + encryptCode, err := getEncKey() + if err != nil { + return err + } + if len(fieldName) > 0 { + decryptStr, err := core.Decrypt(encryptCode, dataVal.Elem().FieldByName(fieldName).String()) + if err != nil { + return err + } + dataVal.Elem().FieldByName(fieldName).Set(reflect.ValueOf(decryptStr)) + } + return nil +} + +func doEncrypt(dataVal reflect.Value, fieldName string) error { + encryptCode, err := getEncKey() + if err != nil { + return err + } + if len(fieldName) > 0 { + plainPwd := dataVal.Elem().FieldByName(fieldName).String() + encyptedStr, err := core.Encrypt(encryptCode, plainPwd) + + if err != nil { + return err + } + dataVal.Elem().FieldByName(fieldName).Set(reflect.ValueOf(encyptedStr)) + } + return nil +} diff --git a/plugins/jira/api/connection.go b/plugins/jira/api/connection.go index 96970d77..aa94537a 100644 --- a/plugins/jira/api/connection.go +++ b/plugins/jira/api/connection.go @@ -25,9 +25,6 @@ import ( "strings" "time" - "github.com/apache/incubator-devlake/config" - "github.com/apache/incubator-devlake/models/common" - "github.com/apache/incubator-devlake/errors" "github.com/apache/incubator-devlake/plugins/core" "github.com/apache/incubator-devlake/plugins/helper" @@ -52,11 +49,12 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro if err != nil { return nil, err } + // test connection apiClient, err := helper.NewApiClient( connection.Endpoint, map[string]string{ - "Authorization": fmt.Sprintf("Basic %v", connection.Auth), + "Authorization": fmt.Sprintf("Basic %v", connection.GetEncodedToken()), }, 3*time.Second, connection.Proxy, @@ -100,111 +98,6 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro return nil, nil } -func findConnectionByInputParam(input *core.ApiResourceInput) (*models.JiraConnection, error) { - jiraConnectionId, err := getJiraConnectionIdByInputParam(input) - if err != nil { - return nil, fmt.Errorf("invalid connectionId") - } - return getJiraConnectionById(jiraConnectionId) -} - -func getJiraConnectionIdByInputParam(input *core.ApiResourceInput) (uint64, error) { - connectionId := input.Params["connectionId"] - if connectionId == "" { - return 0, fmt.Errorf("missing connectionId") - } - return strconv.ParseUint(connectionId, 10, 64) -} - -func getJiraConnectionById(id uint64) (*models.JiraConnection, error) { - jiraConnection := &models.JiraConnection{} - err := db.First(jiraConnection, id).Error - if err != nil { - return nil, err - } - - // decrypt - v := config.GetConfig() - encKey := v.GetString(core.EncodeKeyEnvStr) - jiraConnection.BasicAuthEncoded, err = core.Decrypt(encKey, jiraConnection.BasicAuthEncoded) - if err != nil { - log.Error("failed to decrypt basic auth: %s", err) - } - - return jiraConnection, nil -} -func mergeFieldsToJiraConnection(jiraConnection *models.JiraConnection, connections ...map[string]interface{}) error { - // decode - for _, connection := range connections { - err := mapstructure.Decode(connection, jiraConnection) - if err != nil { - return err - } - } - - // validate - vld := validator.New() - err := vld.Struct(jiraConnection) - if err != nil { - return err - } - - return nil -} - -func refreshAndSaveJiraConnection(jiraConnection *models.JiraConnection, data map[string]interface{}) error { - var err error - // update fields from request body - err = mergeFieldsToJiraConnection(jiraConnection, data) - if err != nil { - return err - } - - encKey, err := getEncKey() - if err != nil { - return err - } - jiraConnection.BasicAuthEncoded, err = core.Encrypt(encKey, jiraConnection.BasicAuthEncoded) - if err != nil { - return err - } - - // transaction for nested operations - tx := db.Begin() - defer func() { - if err != nil { - tx.Rollback() - } else { - tx.Commit() - } - }() - if jiraConnection.ID > 0 { - err = tx.Save(jiraConnection).Error - } else { - err = tx.Create(jiraConnection).Error - } - if err != nil { - if common.IsDuplicateError(err) { - return fmt.Errorf("jira connection with name %s already exists", jiraConnection.Name) - } - return err - } - // perform optional operation - typeMappings := data["typeMappings"] - if typeMappings != nil { - err = saveTypeMappings(tx, jiraConnection.ID, typeMappings) - if err != nil { - return err - } - } - - jiraConnection.BasicAuthEncoded, err = core.Decrypt(encKey, jiraConnection.BasicAuthEncoded) - if err != nil { - log.Error("failed to decrypt basic auth: %s", err) - } - return nil -} - /* POST /plugins/jira/connections { @@ -230,7 +123,7 @@ func PostConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, err jiraConnection := &models.JiraConnection{} // update from request and save to database - err := refreshAndSaveJiraConnection(jiraConnection, input.Body) + err := helper.RefreshAndSaveConnection(jiraConnection, input.Body, db) if err != nil { return nil, err } @@ -259,14 +152,15 @@ PATCH /plugins/jira/connections/:connectionId } */ func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { + jiraConnection := &models.JiraConnection{} // load from db - jiraConnection, err := findConnectionByInputParam(input) + err := helper.FindConnectionByInput(input, jiraConnection, db) if err != nil { return nil, err } // update from request and save to database - err = refreshAndSaveJiraConnection(jiraConnection, input.Body) + err = helper.RefreshAndSaveConnection(jiraConnection, input.Body, db) if err != nil { return nil, err } @@ -279,7 +173,7 @@ DELETE /plugins/jira/connections/:connectionId */ func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { // load from db - jiraConnectionID, err := getJiraConnectionIdByInputParam(input) + jiraConnectionID, err := helper.GetConnectionIdByInputParam(input) if err != nil { return nil, err } @@ -304,22 +198,19 @@ func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, er GET /plugins/jira/connections */ func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { - jiraConnections := make([]models.JiraConnection, 0) + jiraConnections := make([]*models.JiraConnection, 0) err := db.Find(&jiraConnections).Error if err != nil { return nil, err } - encKey, err := getEncKey() - if err != nil { - return nil, err - } - for i := range jiraConnections { - jiraConnections[i].BasicAuthEncoded, err = core.Decrypt(encKey, jiraConnections[i].BasicAuthEncoded) + for i, _ := range jiraConnections { + err = helper.DecryptConnection(jiraConnections[i], "Password") if err != nil { - log.Error("failed to decrypt basic auth: %s", err) + return nil, err } } - return &core.ApiResourceOutput{Body: jiraConnections}, nil + + return &core.ApiResourceOutput{Body: jiraConnections, Status: http.StatusOK}, nil } /* @@ -345,53 +236,27 @@ GET /plugins/jira/connections/:connectionId } */ func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { - jiraConnection, err := findConnectionByInputParam(input) + jiraConnection := &models.JiraConnection{} + err := helper.FindConnectionByInput(input, jiraConnection, db) if err != nil { return nil, err } detail := &models.JiraConnectionDetail{ JiraConnection: *jiraConnection, - TypeMappings: make(map[string]map[string]interface{}), } - typeMappings, err := findIssueTypeMappingByConnectionId(jiraConnection.ID) if err != nil { return nil, err } - for _, jiraTypeMapping := range typeMappings { - // type mapping - typeMappingDict := map[string]interface{}{ - "standardType": jiraTypeMapping.StandardType, - } - detail.TypeMappings[jiraTypeMapping.UserType] = typeMappingDict - - // status mapping - statusMappings, err := findIssueStatusMappingByConnectionIdAndUserType( - jiraConnection.ID, - jiraTypeMapping.UserType, - ) - if err != nil { - return nil, err - } - if len(statusMappings) == 0 { - continue - } - statusMappingsDict := make(map[string]interface{}) - for _, jiraStatusMapping := range statusMappings { - statusMappingsDict[jiraStatusMapping.UserStatus] = map[string]interface{}{ - "standardStatus": jiraStatusMapping.StandardStatus, - } - } - typeMappingDict["statusMappings"] = statusMappingsDict - } return &core.ApiResourceOutput{Body: detail}, nil } // GET /plugins/jira/connections/:connectionId/epics func GetEpicsByConnectionId(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { - jiraConnection, err := findConnectionByInputParam(input) + jiraConnection := &models.JiraConnection{} + err := helper.FindConnectionByInput(input, jiraConnection, db) if err != nil { return nil, err } @@ -410,7 +275,11 @@ type GranularitiesResponse struct { } func GetGranularitiesByConnectionId(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { - jiraConnection, err := findConnectionByInputParam(input) + jiraConnection := &models.JiraConnection{} + err := helper.FindConnectionByInput(input, jiraConnection, db) + if err != nil { + return nil, err + } if err != nil { return nil, err } @@ -448,19 +317,3 @@ func GetBoardsByConnectionId(input *core.ApiResourceInput) (*core.ApiResourceOut } return &core.ApiResourceOutput{Body: boardResponses}, nil } - -func getEncKey() (string, error) { - // encrypt - v := config.GetConfig() - encKey := v.GetString(core.EncodeKeyEnvStr) - if encKey == "" { - // Randomly generate a bunch of encryption keys and set them to config - encKey = core.RandomEncKey() - v.Set(core.EncodeKeyEnvStr, encKey) - err := config.WriteConfig(v) - if err != nil { - return encKey, err - } - } - return encKey, nil -} diff --git a/plugins/jira/api/issue_status_mapping.go b/plugins/jira/api/issue_status_mapping.go deleted file mode 100644 index 9caa90b3..00000000 --- a/plugins/jira/api/issue_status_mapping.go +++ /dev/null @@ -1,216 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package api - -import ( - "fmt" - "net/http" - - "github.com/apache/incubator-devlake/models/common" - - "github.com/apache/incubator-devlake/plugins/core" - "github.com/apache/incubator-devlake/plugins/jira/models" - "github.com/go-playground/validator/v10" - "github.com/mitchellh/mapstructure" - "gorm.io/gorm" -) - -func findIssueStatusMappingFromInput(input *core.ApiResourceInput) (*models.JiraIssueStatusMapping, error) { - // load type mapping - jiraIssueTypeMapping, err := findIssueTypeMappingByInputParam(input) - if err != nil { - return nil, err - } - // load status mapping from db - userStatus := input.Params["userStatus"] - if userStatus == "" { - return nil, fmt.Errorf("missing userStatus") - } - jiraIssueStatusMapping := &models.JiraIssueStatusMapping{} - err = db.First( - jiraIssueStatusMapping, - jiraIssueTypeMapping.ConnectionID, - jiraIssueTypeMapping.UserType, - userStatus, - ).Error - if err != nil { - return nil, err - } - - return jiraIssueStatusMapping, nil -} - -func mergeFieldsToJiraStatusMapping( - jiraIssueStatusMapping *models.JiraIssueStatusMapping, - connections ...map[string]interface{}, -) error { - // merge fields from connections to jiraIssueStatusMapping - for _, connection := range connections { - err := mapstructure.Decode(connection, jiraIssueStatusMapping) - if err != nil { - return err - } - } - // validate - vld := validator.New() - err := vld.Struct(jiraIssueStatusMapping) - if err != nil { - return err - } - return nil -} - -func wrapIssueStatusDuplicateErr(err error) error { - if common.IsDuplicateError(err) { - return fmt.Errorf("jira issue status mapping already exists") - } - return err -} - -func saveStatusMappings(tx *gorm.DB, jiraConnectionId uint64, userType string, statusMappings interface{}) error { - statusMappingsMap, ok := statusMappings.(map[string]interface{}) - if !ok { - return fmt.Errorf("statusMappings is not a JSON object: %v", statusMappings) - } - err := tx.Where( - "connection_id = ? AND user_type = ?", - jiraConnectionId, - userType).Delete(&models.JiraIssueStatusMapping{}).Error - if err != nil { - return err - } - for userStatus, statusMapping := range statusMappingsMap { - statusMappingMap, ok := statusMapping.(map[string]interface{}) - if !ok { - return fmt.Errorf("statusMapping is not a JSON object: %v", statusMappings) - } - jiraIssueStatusMapping := &models.JiraIssueStatusMapping{} - err = mergeFieldsToJiraStatusMapping(jiraIssueStatusMapping, statusMappingMap, map[string]interface{}{ - "ConnectionID": jiraConnectionId, - "UserType": userType, - "UserStatus": userStatus, - }) - if err != nil { - return err - } - err = tx.Create(jiraIssueStatusMapping).Error - if err != nil { - return err - } - } - return nil -} - -func findIssueStatusMappingByConnectionIdAndUserType( - jiraConnectionId uint64, - userType string, -) ([]*models.JiraIssueStatusMapping, error) { - jiraIssueStatusMappings := make([]*models.JiraIssueStatusMapping, 0) - err := db.Where( - "connection_id = ? AND user_type = ?", - jiraConnectionId, - userType, - ).Find(&jiraIssueStatusMappings).Error - return jiraIssueStatusMappings, err -} - -/* -POST /plugins/jira/connections/:connectionId/type-mappings/:userType/status-mappings -{ - "userStatus": "user custom status", - "standardStatus": "devlake standard status" -} -*/ -func PostIssueStatusMappings(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { - jiraIssueTypeMapping, err := findIssueTypeMappingByInputParam(input) - if err != nil { - return nil, err - } - jiraIssueStatusMapping := &models.JiraIssueStatusMapping{} - err = mergeFieldsToJiraStatusMapping(jiraIssueStatusMapping, input.Body, map[string]interface{}{ - "ConnectionID": jiraIssueTypeMapping.ConnectionID, - "UserType": jiraIssueTypeMapping.UserType, - }) - if err != nil { - return nil, err - } - // save - err = wrapIssueStatusDuplicateErr(db.Create(jiraIssueStatusMapping).Error) - if err != nil { - return nil, err - } - return &core.ApiResourceOutput{Body: jiraIssueStatusMapping, Status: http.StatusOK}, nil -} - -/* -PUT /plugins/jira/connections/:connectionId/type-mappings/:userType/status-mappings/:userStatus -{ - "standardStatus": "devlake standard status" -} -*/ -func PutIssueStatusMapping(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { - // load from db - jiraIssueStatusMapping, err := findIssueStatusMappingFromInput(input) - if err != nil { - return nil, err - } - // update with request body - err = mergeFieldsToJiraStatusMapping(jiraIssueStatusMapping, input.Body) - if err != nil { - return nil, err - } - // save - err = wrapIssueStatusDuplicateErr(db.Save(jiraIssueStatusMapping).Error) - if err != nil { - return nil, err - } - return &core.ApiResourceOutput{Body: jiraIssueStatusMapping}, nil -} - -/* -DELETE /plugins/jira/connections/:connectionId/type-mappings/:userType/status-mappings/:userStatus -*/ -func DeleteIssueStatusMapping(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { - jiraIssueStatusMapping, err := findIssueStatusMappingFromInput(input) - if err != nil { - return nil, err - } - err = db.Delete(jiraIssueStatusMapping).Error - if err != nil { - return nil, err - } - return &core.ApiResourceOutput{Body: jiraIssueStatusMapping}, nil -} - -/* -GET /plugins/jira/connections/:connectionId/type-mappings/:userType/status-mappings -*/ -func ListIssueStatusMappings(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { - jiraIssueTypeMapping, err := findIssueTypeMappingByInputParam(input) - if err != nil { - return nil, err - } - jiraIssueStatusMappings, err := findIssueStatusMappingByConnectionIdAndUserType( - jiraIssueTypeMapping.ConnectionID, - jiraIssueTypeMapping.UserType, - ) - if err != nil { - return nil, err - } - return &core.ApiResourceOutput{Body: jiraIssueStatusMappings}, nil -} diff --git a/plugins/jira/api/issue_type_mapping.go b/plugins/jira/api/issue_type_mapping.go deleted file mode 100644 index 7f135646..00000000 --- a/plugins/jira/api/issue_type_mapping.go +++ /dev/null @@ -1,214 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package api - -import ( - "fmt" - "net/http" - - "github.com/apache/incubator-devlake/models/common" - - "github.com/apache/incubator-devlake/plugins/core" - "github.com/apache/incubator-devlake/plugins/jira/models" - "github.com/go-playground/validator/v10" - "github.com/mitchellh/mapstructure" - "gorm.io/gorm" -) - -func findIssueTypeMappingByInputParam(input *core.ApiResourceInput) (*models.JiraIssueTypeMapping, error) { - // load jira connection - jiraConnection, err := findConnectionByInputParam(input) - if err != nil { - return nil, err - } - // load jira type mapping from db - userType := input.Params["userType"] - if userType == "" { - return nil, fmt.Errorf("missing userType") - } - jiraIssueTypeMapping := &models.JiraIssueTypeMapping{} - err = db.First(jiraIssueTypeMapping, jiraConnection.ID, userType).Error - if err != nil { - return nil, err - } - - return jiraIssueTypeMapping, nil -} - -func mergeFieldsToJiraTypeMapping( - jiraIssueTypeMapping *models.JiraIssueTypeMapping, - connections ...map[string]interface{}, -) error { - // merge fields from connections to jiraIssueTypeMapping - for _, connection := range connections { - err := mapstructure.Decode(connection, jiraIssueTypeMapping) - if err != nil { - return err - } - } - // validate - vld := validator.New() - err := vld.Struct(jiraIssueTypeMapping) - if err != nil { - return err - } - return nil -} - -func wrapIssueTypeDuplicateErr(err error) error { - if common.IsDuplicateError(err) { - return fmt.Errorf("jira issue type mapping already exists") - } - return err -} - -func saveTypeMappings(tx *gorm.DB, jiraConnectionId uint64, typeMappings interface{}) error { - typeMappingsMap, ok := typeMappings.(map[string]interface{}) - if !ok { - return fmt.Errorf("typeMappings is not a JSON object: %v", typeMappings) - } - err := tx.Where("connection_id = ?", jiraConnectionId).Delete(&models.JiraIssueTypeMapping{}).Error - if err != nil { - return err - } - for userType, typeMapping := range typeMappingsMap { - typeMappingMap, ok := typeMapping.(map[string]interface{}) - if !ok { - return fmt.Errorf("typeMapping is not a JSON object: %v", typeMapping) - } - jiraIssueTypeMapping := &models.JiraIssueTypeMapping{} - err = mergeFieldsToJiraTypeMapping(jiraIssueTypeMapping, typeMappingMap, map[string]interface{}{ - "ConnectionID": jiraConnectionId, - "UserType": userType, - }) - if err != nil { - return err - } - err = wrapIssueTypeDuplicateErr(tx.Create(jiraIssueTypeMapping).Error) - if err != nil { - return err - } - - statusMappings := typeMappingMap["statusMappings"] - if statusMappings != nil { - err = saveStatusMappings(tx, jiraConnectionId, userType, statusMappings) - if err != nil { - return err - } - } - } - return nil -} - -func findIssueTypeMappingByConnectionId(jiraConnectionId uint64) ([]*models.JiraIssueTypeMapping, error) { - jiraIssueTypeMappings := make([]*models.JiraIssueTypeMapping, 0) - err := db.Where("connection_id = ?", jiraConnectionId).Find(&jiraIssueTypeMappings).Error - if err != nil { - return nil, err - } - return jiraIssueTypeMappings, nil -} - -/* -POST /plugins/jira/connections/:connectionId/type-mappings -{ - "userType": "user custom type", - "standardType": "devlake standard type" -} -*/ -func PostIssueTypeMappings(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { - // create new - jiraConnection, err := findConnectionByInputParam(input) - if err != nil { - return nil, err - } - jiraIssueTypeMapping := &models.JiraIssueTypeMapping{} - err = mergeFieldsToJiraTypeMapping(jiraIssueTypeMapping, input.Body, map[string]interface{}{ - "ConnectionID": jiraConnection.ID, - }) - if err != nil { - return nil, err - } - // save - err = wrapIssueTypeDuplicateErr(db.Create(jiraIssueTypeMapping).Error) - if err != nil { - return nil, err - } - return &core.ApiResourceOutput{Body: jiraIssueTypeMapping, Status: http.StatusOK}, nil -} - -/* -PUT /plugins/jira/connections/:connectionId/type-mappings/:userType -{ - "standardType": "devlake standard type" -} -*/ -func PutIssueTypeMapping(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { - // load from db - jiraIssueTypeMapping, err := findIssueTypeMappingByInputParam(input) - if err != nil { - return nil, err - } - // update with request body - err = mergeFieldsToJiraTypeMapping(jiraIssueTypeMapping, input.Body) - if err != nil { - return nil, err - } - // save - err = wrapIssueTypeDuplicateErr(db.Save(jiraIssueTypeMapping).Error) - if err != nil { - return nil, err - } - return &core.ApiResourceOutput{Body: jiraIssueTypeMapping}, nil -} - -/* -DELETE /plugins/jira/connections/:connectionId/type-mappings/:userType -*/ -func DeleteIssueTypeMapping(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { - // load from db - jiraIssueTypeMapping, err := findIssueTypeMappingByInputParam(input) - if err != nil { - return nil, err - } - err = db.Delete(jiraIssueTypeMapping).Error - if err != nil { - return nil, err - } - err = db.Where( - "connection_id = ? AND user_type = ?", - jiraIssueTypeMapping.ConnectionID, - jiraIssueTypeMapping.UserType, - ).Delete(&models.JiraIssueStatusMapping{}).Error - if err != nil { - return nil, err - } - return &core.ApiResourceOutput{Body: jiraIssueTypeMapping}, nil -} - -/* -GET /plugins/jira/connections/:connectionId/type-mappings -*/ -func ListIssueTypeMappings(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { - jiraConnection, err := findConnectionByInputParam(input) - if err != nil { - return nil, err - } - jiraIssueTypeMappings, err := findIssueTypeMappingByConnectionId(jiraConnection.ID) - return &core.ApiResourceOutput{Body: jiraIssueTypeMappings}, err -} diff --git a/plugins/jira/api/proxy.go b/plugins/jira/api/proxy.go index 55331e01..e4a3eb90 100644 --- a/plugins/jira/api/proxy.go +++ b/plugins/jira/api/proxy.go @@ -20,8 +20,8 @@ package api import ( "encoding/json" "fmt" + "github.com/apache/incubator-devlake/utils" "io/ioutil" - "strconv" "time" "github.com/apache/incubator-devlake/plugins/core" @@ -34,21 +34,13 @@ const ( ) func Proxy(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { - connectionId := input.Params["connectionId"] - if connectionId == "" { - return nil, fmt.Errorf("missing connectionid") - } - jiraConnectionId, err := strconv.ParseUint(connectionId, 10, 64) - if err != nil { - return nil, err - } jiraConnection := &models.JiraConnection{} - err = db.First(jiraConnection, jiraConnectionId).Error + err := helper.FindConnectionByInput(input, jiraConnection, db) if err != nil { return nil, err } - encKey := cfg.GetString(core.EncodeKeyEnvStr) - basicAuth, err := core.Decrypt(encKey, jiraConnection.BasicAuthEncoded) + basicAuth := utils.GetEncodedToken(jiraConnection.Username, jiraConnection.Password) + if err != nil { return nil, err } diff --git a/plugins/jira/jira.go b/plugins/jira/jira.go index 2b865ea6..2a40b901 100644 --- a/plugins/jira/jira.go +++ b/plugins/jira/jira.go @@ -150,6 +150,7 @@ func (plugin Jira) MigrationScripts() []migration.Script { new(migrationscripts.UpdateSchemas20220505), new(migrationscripts.UpdateSchemas20220507), new(migrationscripts.UpdateSchemas20220518), + new(migrationscripts.UpdateSchemas20220524), new(migrationscripts.UpdateSchemas20220525), new(migrationscripts.UpdateSchemas20220526), new(migrationscripts.UpdateSchemas20220527), @@ -184,22 +185,6 @@ func (plugin Jira) ApiResources() map[string]map[string]core.ApiResourceHandler "connections/:connectionId/boards": { "GET": api.GetBoardsByConnectionId, }, - "connections/:connectionId/type-mappings": { - "POST": api.PostIssueTypeMappings, - "GET": api.ListIssueTypeMappings, - }, - "connections/:connectionId/type-mappings/:userType": { - "PUT": api.PutIssueTypeMapping, - "DELETE": api.DeleteIssueTypeMapping, - }, - "connections/:connectionId/type-mappings/:userType/status-mappings": { - "POST": api.PostIssueStatusMappings, - "GET": api.ListIssueStatusMappings, - }, - "connections/:connectionId/type-mappings/:userType/status-mappings/:userStatus": { - "PUT": api.PutIssueStatusMapping, - "DELETE": api.DeleteIssueStatusMapping, - }, "connections/:connectionId/proxy/rest/*path": { "GET": api.Proxy, }, diff --git a/plugins/jira/models/connection.go b/plugins/jira/models/connection.go index edb6b88f..f1ed2f8a 100644 --- a/plugins/jira/models/connection.go +++ b/plugins/jira/models/connection.go @@ -18,7 +18,7 @@ limitations under the License. package models import ( - "github.com/apache/incubator-devlake/models/common" + "github.com/apache/incubator-devlake/plugins/helper" ) type EpicResponse struct { @@ -28,9 +28,9 @@ type EpicResponse struct { } type TestConnectionRequest struct { - Endpoint string `json:"endpoint"` - Auth string `json:"auth"` - Proxy string `json:"proxy"` + Endpoint string `json:"endpoint"` + Proxy string `json:"proxy"` + helper.BasicAuth `mapstructure:",squash"` } type BoardResponse struct { @@ -40,15 +40,11 @@ type BoardResponse struct { } type JiraConnection struct { - common.Model - Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"` - Endpoint string `json:"endpoint" validate:"required"` - BasicAuthEncoded string `json:"basicAuthEncoded" validate:"required"` + helper.RestConnection `mapstructure:",squash"` + helper.BasicAuth `mapstructure:",squash"` EpicKeyField string `gorm:"type:varchar(50);" json:"epicKeyField"` StoryPointField string `gorm:"type:varchar(50);" json:"storyPointField"` RemotelinkCommitShaPattern string `gorm:"type:varchar(255);comment='golang regexp, the first group will be recognized as commit sha, ref https://github.com/google/re2/wiki/Syntax'" json:"remotelinkCommitShaPattern"` - Proxy string `json:"proxy"` - RateLimit int `comment:"api request rate limt per hour" json:"rateLimit"` } type JiraIssueTypeMapping struct { diff --git a/plugins/jira/models/migrationscripts/updateSchemas20220505.go b/plugins/jira/models/migrationscripts/updateSchemas20220505.go index 8e0e1610..ff4a6c81 100644 --- a/plugins/jira/models/migrationscripts/updateSchemas20220505.go +++ b/plugins/jira/models/migrationscripts/updateSchemas20220505.go @@ -19,16 +19,32 @@ package migrationscripts import ( "context" - "github.com/apache/incubator-devlake/plugins/jira/models" + "github.com/apache/incubator-devlake/models/common" "github.com/apache/incubator-devlake/plugins/jira/models/migrationscripts/archived" "gorm.io/gorm" ) +type JiraConnection20220505 struct { + common.Model + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"` + Endpoint string `json:"endpoint" validate:"required"` + BasicAuthEncoded string `json:"basicAuthEncoded" validate:"required"` + EpicKeyField string `gorm:"type:varchar(50);" json:"epicKeyField"` + StoryPointField string `gorm:"type:varchar(50);" json:"storyPointField"` + RemotelinkCommitShaPattern string `gorm:"type:varchar(255);comment='golang regexp, the first group will be recognized as commit sha, ref https://github.com/google/re2/wiki/Syntax'" json:"remotelinkCommitShaPattern"` + Proxy string `json:"proxy"` + RateLimit int `comment:"api request rate limt per hour" json:"rateLimit"` +} + +func (JiraConnection20220505) TableName() string { + return "_tool_jira_connections" +} + type UpdateSchemas20220505 struct{} func (*UpdateSchemas20220505) Up(ctx context.Context, db *gorm.DB) error { - err := db.Migrator().RenameTable(archived.JiraSource{}, models.JiraConnection{}) + err := db.Migrator().RenameTable(archived.JiraSource{}, JiraConnection20220505{}) if err != nil { return err } diff --git a/plugins/jira/models/migrationscripts/updateSchemas20220524.go b/plugins/jira/models/migrationscripts/updateSchemas20220524.go new file mode 100644 index 00000000..9c753610 --- /dev/null +++ b/plugins/jira/models/migrationscripts/updateSchemas20220524.go @@ -0,0 +1,72 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "context" + "github.com/apache/incubator-devlake/plugins/helper" + "gorm.io/gorm" +) + +type JiraConnection20220524 struct { + helper.RestConnection + helper.BasicAuth + EpicKeyField string `gorm:"type:varchar(50);" json:"epicKeyField"` + StoryPointField string `gorm:"type:varchar(50);" json:"storyPointField"` + RemotelinkCommitShaPattern string `gorm:"type:varchar(255);comment='golang regexp, the first group will be recognized as commit sha, ref https://github.com/google/re2/wiki/Syntax'" json:"remotelinkCommitShaPattern"` +} + +func (JiraConnection20220524) TableName() string { + return "_tool_jira_connections" +} + +type UpdateSchemas20220524 struct{} + +func (*UpdateSchemas20220524) Up(ctx context.Context, db *gorm.DB) error { + var err error + if !db.Migrator().HasColumn(&JiraConnection20220505{}, "password") { + err = db.Migrator().AddColumn(&JiraConnection20220524{}, "password") + if err != nil { + return err + } + } + + if !db.Migrator().HasColumn(&JiraConnection20220505{}, "username") { + err = db.Migrator().AddColumn(&JiraConnection20220524{}, "username") + if err != nil { + return err + } + } + + if db.Migrator().HasColumn(&JiraConnection20220505{}, "basic_auth_encoded") { + err = db.Migrator().DropColumn(&JiraConnection20220505{}, "basic_auth_encoded") + if err != nil { + return err + } + } + + return nil +} + +func (*UpdateSchemas20220524) Version() uint64 { + return 20220507154646 +} + +func (*UpdateSchemas20220524) Name() string { + return "Add icon_url column to JiraIssue" +} diff --git a/plugins/jira/tasks/api_client.go b/plugins/jira/tasks/api_client.go index 4896c05a..3673bee5 100644 --- a/plugins/jira/tasks/api_client.go +++ b/plugins/jira/tasks/api_client.go @@ -29,10 +29,12 @@ import ( func NewJiraApiClient(taskCtx core.TaskContext, connection *models.JiraConnection) (*helper.ApiAsyncClient, error) { // load configuration encKey := taskCtx.GetConfig(core.EncodeKeyEnvStr) - auth, err := core.Decrypt(encKey, connection.BasicAuthEncoded) + decodedPassword, err := core.Decrypt(encKey, connection.Password) if err != nil { - return nil, fmt.Errorf("Failed to decrypt Auth Token: %w", err) + return nil, fmt.Errorf("Failed to decrypt Auth AccessToken: %w", err) } + connection.Password = decodedPassword + auth := connection.GetEncodedToken() // create synchronize api client so we can calculate api rate limit dynamically headers := map[string]string{ @@ -44,7 +46,7 @@ func NewJiraApiClient(taskCtx core.TaskContext, connection *models.JiraConnectio } apiClient.SetAfterFunction(func(res *http.Response) error { if res.StatusCode == http.StatusUnauthorized { - return fmt.Errorf("authentication failed, please check your Basic Auth Token") + return fmt.Errorf("authentication failed, please check your AccessToken") } return nil }) diff --git a/plugins/tapd/tasks/api_client.go b/plugins/tapd/tasks/api_client.go index 406bc97a..977d076f 100644 --- a/plugins/tapd/tasks/api_client.go +++ b/plugins/tapd/tasks/api_client.go @@ -31,7 +31,7 @@ func NewTapdApiClient(taskCtx core.TaskContext, connection *models.TapdConnectio encKey := taskCtx.GetConfig(core.EncodeKeyEnvStr) auth, err := core.Decrypt(encKey, connection.BasicAuthEncoded) if err != nil { - return nil, fmt.Errorf("Failed to decrypt Auth Token: %w", err) + return nil, fmt.Errorf("Failed to decrypt Auth AccessToken: %w", err) } // create synchronize api client so we can calculate api rate limit dynamically @@ -44,7 +44,7 @@ func NewTapdApiClient(taskCtx core.TaskContext, connection *models.TapdConnectio } apiClient.SetAfterFunction(func(res *http.Response) error { if res.StatusCode == http.StatusUnprocessableEntity { - return fmt.Errorf("authentication failed, please check your Basic Auth Token") + return fmt.Errorf("authentication failed, please check your AccessToken") } return nil })
