This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 9c00c4de Feishu plugin support multi-connections. (#2322)
9c00c4de is described below
commit 9c00c4de92fff2e96a6e744792c860a7d7d86a2f
Author: likyh <[email protected]>
AuthorDate: Mon Jun 27 14:20:38 2022 +0800
Feishu plugin support multi-connections. (#2322)
* Feishu plugin support multi-connections.
* fix bug
* fix a bug and ci
Co-authored-by: linyh <[email protected]>
---
plugins/feishu/api/connection.go | 133 +++++++++++++++++++++
plugins/feishu/{tasks/task_data.go => api/init.go} | 29 ++---
.../register.go => apimodels/access_token.go} | 20 ++--
plugins/feishu/e2e/meeting_test.go | 61 ++++++++++
.../_raw_feishu_meeting_top_user_item.csv | 29 +++++
.../_tool_feishu_meeting_top_user_items.csv | 29 +++++
plugins/feishu/feishu.go | 74 ++----------
plugins/feishu/{feishu.go => impl/impl.go} | 72 +++++++----
.../{meeting_top_user_item.go => connection.go} | 26 ++--
plugins/feishu/models/meeting_top_user_item.go | 1 +
.../{init_schema.go => archived/connection.go} | 29 ++---
.../archived/meeting_top_user_item.go | 17 ++-
.../feishu/models/migrationscripts/init_schema.go | 45 ++++++-
plugins/feishu/models/migrationscripts/register.go | 2 +-
.../migrationscripts/updateSchemas20220526.go | 90 --------------
plugins/feishu/tasks/api_client.go | 47 ++------
.../tasks/meeting_top_user_item_collector.go | 5 +-
.../tasks/meeting_top_user_item_extractor.go | 6 +-
plugins/feishu/tasks/task_data.go | 6 +-
.../e2e/snapshot_tables/issue_commits_story.csv | 2 +
20 files changed, 435 insertions(+), 288 deletions(-)
diff --git a/plugins/feishu/api/connection.go b/plugins/feishu/api/connection.go
new file mode 100644
index 00000000..a88e84fb
--- /dev/null
+++ b/plugins/feishu/api/connection.go
@@ -0,0 +1,133 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/plugins/feishu/apimodels"
+ "github.com/apache/incubator-devlake/plugins/feishu/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/mitchellh/mapstructure"
+ "net/http"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+)
+
+/*
+POST /plugins/feishu/test
+*/
+func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput,
error) {
+ // process input
+ var params models.TestConnectionRequest
+ err := mapstructure.Decode(input.Body, ¶ms)
+ if err != nil {
+ return nil, err
+ }
+ err = vld.Struct(params)
+ if err != nil {
+ return nil, err
+ }
+
+ authApiClient, err := helper.NewApiClient(params.Endpoint, nil, 0,
params.Proxy, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ // request for access token
+ tokenReqBody := &apimodels.ApiAccessTokenRequest{
+ AppId: params.AppId,
+ AppSecret: params.SecretKey,
+ }
+ tokenRes, err :=
authApiClient.Post("open-apis/auth/v3/tenant_access_token/internal", nil,
tokenReqBody, nil)
+ if err != nil {
+ return nil, err
+ }
+ tokenResBody := &apimodels.ApiAccessTokenResponse{}
+ err = helper.UnmarshalResponse(tokenRes, tokenResBody)
+ if err != nil {
+ return nil, err
+ }
+ if tokenResBody.AppAccessToken == "" && tokenResBody.TenantAccessToken
== "" {
+ return nil, fmt.Errorf("failed to request access token")
+ }
+
+ // output
+ return nil, nil
+}
+
+/*
+POST /plugins/feishu/connections
+*/
+func PostConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput,
error) {
+ connection := &models.FeishuConnection{}
+ err := connectionHelper.Create(connection, input)
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: connection, Status:
http.StatusOK}, nil
+}
+
+/*
+PATCH /plugins/feishu/connections/:connectionId
+*/
+func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput,
error) {
+ connection := &models.FeishuConnection{}
+ err := connectionHelper.Patch(connection, input)
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: connection, Status:
http.StatusOK}, nil
+}
+
+/*
+DELETE /plugins/feishu/connections/:connectionId
+*/
+func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput,
error) {
+ connection := &models.FeishuConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+ err = connectionHelper.Delete(connection)
+ return &core.ApiResourceOutput{Body: connection}, err
+}
+
+/*
+GET /plugins/feishu/connections
+*/
+func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput,
error) {
+ var connections []models.FeishuConnection
+ err := connectionHelper.List(&connections)
+ if err != nil {
+ return nil, err
+ }
+
+ return &core.ApiResourceOutput{Body: connections}, nil
+}
+
+/*
+GET /plugins/feishu/connections/:connectionId
+*/
+func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput,
error) {
+ connection := &models.FeishuConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: connection}, err
+}
diff --git a/plugins/feishu/tasks/task_data.go b/plugins/feishu/api/init.go
similarity index 63%
copy from plugins/feishu/tasks/task_data.go
copy to plugins/feishu/api/init.go
index aad0549d..6774e148 100644
--- a/plugins/feishu/tasks/task_data.go
+++ b/plugins/feishu/api/init.go
@@ -15,24 +15,25 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package tasks
+package api
import (
- "github.com/apache/incubator-devlake/plugins/feishu/models"
+ "github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/go-playground/validator/v10"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
)
-type FeishuApiParams struct {
- ApiResName string `json:"apiResName"`
-}
-
-type FeishuOptions struct {
- NumOfDaysToCollect float64 `json:"numOfDaysToCollect"`
- Tasks []string `json:"tasks,omitempty"`
-}
+var vld *validator.Validate
+var connectionHelper *helper.ConnectionApiHelper
+var basicRes core.BasicRes
-type FeishuTaskData struct {
- Options *FeishuOptions
- ApiClient *helper.ApiAsyncClient
- FeishuMeetingTopUserItem *models.FeishuMeetingTopUserItem
+func Init(config *viper.Viper, logger core.Logger, database *gorm.DB) {
+ basicRes = helper.NewDefaultBasicRes(config, logger, database)
+ vld = validator.New()
+ connectionHelper = helper.NewConnectionHelper(
+ basicRes,
+ vld,
+ )
}
diff --git a/plugins/feishu/models/migrationscripts/register.go
b/plugins/feishu/apimodels/access_token.go
similarity index 66%
copy from plugins/feishu/models/migrationscripts/register.go
copy to plugins/feishu/apimodels/access_token.go
index 57d8ea48..d61f2bf9 100644
--- a/plugins/feishu/models/migrationscripts/register.go
+++ b/plugins/feishu/apimodels/access_token.go
@@ -15,15 +15,17 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package apimodels
-import (
- "github.com/apache/incubator-devlake/migration"
-)
+type ApiAccessTokenRequest struct {
+ AppId string `json:"app_id"`
+ AppSecret string `json:"app_secret"`
+}
-// All return all the migration scripts
-func All() []migration.Script {
- return []migration.Script{
- new(InitSchemas), new(UpdateSchemas20220524),
- }
+type ApiAccessTokenResponse struct {
+ Code int `json:"code"`
+ Msg string `json:"msg"`
+ AppAccessToken string `json:"app_access_token"`
+ TenantAccessToken string `json:"tenant_access_token"`
+ Expire int `json:"expire"`
}
diff --git a/plugins/feishu/e2e/meeting_test.go
b/plugins/feishu/e2e/meeting_test.go
new file mode 100644
index 00000000..a2d3e7c9
--- /dev/null
+++ b/plugins/feishu/e2e/meeting_test.go
@@ -0,0 +1,61 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "testing"
+
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/feishu/impl"
+ "github.com/apache/incubator-devlake/plugins/feishu/models"
+ "github.com/apache/incubator-devlake/plugins/feishu/tasks"
+)
+
+func TestMeetingDataFlow(t *testing.T) {
+ var plugin impl.Feishu
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "feishu", plugin)
+
+ taskData := &tasks.FeishuTaskData{
+ Options: &tasks.FeishuOptions{
+ ConnectionId: 1,
+ },
+ }
+
+ // import raw data table
+
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_feishu_meeting_top_user_item.csv",
"_raw_feishu_meeting_top_user_item")
+
+ // verify extraction
+ dataflowTester.FlushTabler(&models.FeishuMeetingTopUserItem{})
+ dataflowTester.Subtask(tasks.ExtractMeetingTopUserItemMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.FeishuMeetingTopUserItem{},
+ "./snapshot_tables/_tool_feishu_meeting_top_user_items.csv",
+ []string{
+ "connection_id",
+ "start_time",
+ "name",
+ "meeting_count",
+ "meeting_duration",
+ "user_type",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+}
diff --git
a/plugins/feishu/e2e/raw_tables/_raw_feishu_meeting_top_user_item.csv
b/plugins/feishu/e2e/raw_tables/_raw_feishu_meeting_top_user_item.csv
new file mode 100644
index 00000000..9e64c15c
--- /dev/null
+++ b/plugins/feishu/e2e/raw_tables/_raw_feishu_meeting_top_user_item.csv
@@ -0,0 +1,29 @@
+id,params,data,url,input,created_at
+1,"{""connectionId"":1}","{""id"":""ou_e1ed6bb5eb13cad38cfa8531d59cde11"",""meeting_count"":""9"",""meeting_duration"":""256"",""name"":""用户A"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655856000&limit=100&order_by=2&start_time=1655769600,"{""PairEndTime"":
""2022-06-22T08:00:00+08:00"", ""PairStartTime"":
""2022-06-21T08:00:00+08:00""}",2022-06-22 06:24:43.912949+00:00
+2,"{""connectionId"":1}","{""id"":""ou_f72a1e52175e2a1e19bcae48af44d2ed"",""meeting_count"":""7"",""meeting_duration"":""167"",""name"":""用户B"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655856000&limit=100&order_by=2&start_time=1655769600,"{""PairEndTime"":
""2022-06-22T08:00:00+08:00"", ""PairStartTime"":
""2022-06-21T08:00:00+08:00""}",2022-06-22 06:24:43.912949+00:00
+3,"{""connectionId"":1}","{""id"":""ou_78e637b7fc8c614741e412c55e65f46e"",""meeting_count"":""9"",""meeting_duration"":""161"",""name"":""用户C"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655856000&limit=100&order_by=2&start_time=1655769600,"{""PairEndTime"":
""2022-06-22T08:00:00+08:00"", ""PairStartTime"":
""2022-06-21T08:00:00+08:00""}",2022-06-22 06:24:43.912949+00:00
+4,"{""connectionId"":1}","{""id"":""ou_eb39e98fe1cee6ee280274f393242caa"",""meeting_count"":""8"",""meeting_duration"":""151"",""name"":""用户D"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655856000&limit=100&order_by=2&start_time=1655769600,"{""PairEndTime"":
""2022-06-22T08:00:00+08:00"", ""PairStartTime"":
""2022-06-21T08:00:00+08:00""}",2022-06-22 06:24:43.912949+00:00
+5,"{""connectionId"":1}","{""id"":""ou_1754a884a17660b90d9b469e409b5d49"",""meeting_count"":""11"",""meeting_duration"":""136"",""name"":""用户E"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655856000&limit=100&order_by=2&start_time=1655769600,"{""PairEndTime"":
""2022-06-22T08:00:00+08:00"", ""PairStartTime"":
""2022-06-21T08:00:00+08:00""}",2022-06-22 06:24:43.912949+00:00
+6,"{""connectionId"":1}","{""id"":""ou_da0778cf463408f2d66ec13223d4982c"",""meeting_count"":""5"",""meeting_duration"":""126"",""name"":""用户F"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655856000&limit=100&order_by=2&start_time=1655769600,"{""PairEndTime"":
""2022-06-22T08:00:00+08:00"", ""PairStartTime"":
""2022-06-21T08:00:00+08:00""}",2022-06-22 06:24:43.912949+00:00
+7,"{""connectionId"":1}","{""id"":""ou_0d78dea7baacc61b47b4ddbc89f06e98"",""meeting_count"":""5"",""meeting_duration"":""110"",""name"":""用户G"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655856000&limit=100&order_by=2&start_time=1655769600,"{""PairEndTime"":
""2022-06-22T08:00:00+08:00"", ""PairStartTime"":
""2022-06-21T08:00:00+08:00""}",2022-06-22 06:24:43.912949+00:00
+8,"{""connectionId"":1}","{""id"":""ou_fb0302221f0f37d2edf83083908b940a"",""meeting_count"":""3"",""meeting_duration"":""109"",""name"":""用户H"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655856000&limit=100&order_by=2&start_time=1655769600,"{""PairEndTime"":
""2022-06-22T08:00:00+08:00"", ""PairStartTime"":
""2022-06-21T08:00:00+08:00""}",2022-06-22 06:24:43.912949+00:00
+9,"{""connectionId"":1}","{""id"":""ou_e96190d85dd4356083b85d128e5ee6a8"",""meeting_count"":""3"",""meeting_duration"":""104"",""name"":""用户I"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655856000&limit=100&order_by=2&start_time=1655769600,"{""PairEndTime"":
""2022-06-22T08:00:00+08:00"", ""PairStartTime"":
""2022-06-21T08:00:00+08:00""}",2022-06-22 06:24:43.912949+00:00
+10,"{""connectionId"":1}","{""id"":""ou_29dc478a9360719ccc5b820403cec5b5"",""meeting_count"":""7"",""meeting_duration"":""102"",""name"":""用户J"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655856000&limit=100&order_by=2&start_time=1655769600,"{""PairEndTime"":
""2022-06-22T08:00:00+08:00"", ""PairStartTime"":
""2022-06-21T08:00:00+08:00""}",2022-06-22 06:24:43.912949+00:00
+11,"{""connectionId"":1}","{""id"":""ou_5bfe3a304fc59abf04fae8933d94a918"",""meeting_count"":""6"",""meeting_duration"":""102"",""name"":""用户K"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655856000&limit=100&order_by=2&start_time=1655769600,"{""PairEndTime"":
""2022-06-22T08:00:00+08:00"", ""PairStartTime"":
""2022-06-21T08:00:00+08:00""}",2022-06-22 06:24:43.912949+00:00
+12,"{""connectionId"":1}","{""id"":""ou_adcedb2047821324708d95da060d622d"",""meeting_count"":""2"",""meeting_duration"":""97"",""name"":""用户L"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655856000&limit=100&order_by=2&start_time=1655769600,"{""PairEndTime"":
""2022-06-22T08:00:00+08:00"", ""PairStartTime"":
""2022-06-21T08:00:00+08:00""}",2022-06-22 06:24:43.912949+00:00
+13,"{""connectionId"":1}","{""id"":""ou_f519f036e4a8cedd68be360c7f994343"",""meeting_count"":""4"",""meeting_duration"":""97"",""name"":""用户M"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655856000&limit=100&order_by=2&start_time=1655769600,"{""PairEndTime"":
""2022-06-22T08:00:00+08:00"", ""PairStartTime"":
""2022-06-21T08:00:00+08:00""}",2022-06-22 06:24:43.912949+00:00
+14,"{""connectionId"":1}","{""id"":""ou_642d3c32689dd63de1ac1c9c841b7eb8"",""meeting_count"":""2"",""meeting_duration"":""96"",""name"":""用户N"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655856000&limit=100&order_by=2&start_time=1655769600,"{""PairEndTime"":
""2022-06-22T08:00:00+08:00"", ""PairStartTime"":
""2022-06-21T08:00:00+08:00""}",2022-06-22 06:24:43.912949+00:00
+70,"{""connectionId"":1}","{""id"":""ou_e1ed6bb5eb13cad38cfa8531d59cde11"",""meeting_count"":""21"",""meeting_duration"":""706"",""name"":""用户A"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655769600&limit=100&order_by=2&start_time=1655683200,"{""PairEndTime"":
""2022-06-21T08:00:00+08:00"", ""PairStartTime"":
""2022-06-20T08:00:00+08:00""}",2022-06-22 06:24:44.399961+00:00
+71,"{""connectionId"":1}","{""id"":""ou_fb0302221f0f37d2edf83083908b940a"",""meeting_count"":""11"",""meeting_duration"":""456"",""name"":""用户H"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655769600&limit=100&order_by=2&start_time=1655683200,"{""PairEndTime"":
""2022-06-21T08:00:00+08:00"", ""PairStartTime"":
""2022-06-20T08:00:00+08:00""}",2022-06-22 06:24:44.399961+00:00
+72,"{""connectionId"":1}","{""id"":""ou_f72a1e52175e2a1e19bcae48af44d2ed"",""meeting_count"":""13"",""meeting_duration"":""417"",""name"":""用户B"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655769600&limit=100&order_by=2&start_time=1655683200,"{""PairEndTime"":
""2022-06-21T08:00:00+08:00"", ""PairStartTime"":
""2022-06-20T08:00:00+08:00""}",2022-06-22 06:24:44.399961+00:00
+73,"{""connectionId"":1}","{""id"":""ou_d5d7f2df8148ae282544c4c4ad7e0fe0"",""meeting_count"":""9"",""meeting_duration"":""417"",""name"":""用户O"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655769600&limit=100&order_by=2&start_time=1655683200,"{""PairEndTime"":
""2022-06-21T08:00:00+08:00"", ""PairStartTime"":
""2022-06-20T08:00:00+08:00""}",2022-06-22 06:24:44.399961+00:00
+74,"{""connectionId"":1}","{""id"":""ou_1754a884a17660b90d9b469e409b5d49"",""meeting_count"":""17"",""meeting_duration"":""377"",""name"":""用户E"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655769600&limit=100&order_by=2&start_time=1655683200,"{""PairEndTime"":
""2022-06-21T08:00:00+08:00"", ""PairStartTime"":
""2022-06-20T08:00:00+08:00""}",2022-06-22 06:24:44.399961+00:00
+75,"{""connectionId"":1}","{""id"":""ou_f519f036e4a8cedd68be360c7f994343"",""meeting_count"":""12"",""meeting_duration"":""307"",""name"":""用户M"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655769600&limit=100&order_by=2&start_time=1655683200,"{""PairEndTime"":
""2022-06-21T08:00:00+08:00"", ""PairStartTime"":
""2022-06-20T08:00:00+08:00""}",2022-06-22 06:24:44.399961+00:00
+76,"{""connectionId"":1}","{""id"":""ou_32c9b4db9247016c8fd4d45647b39074"",""meeting_count"":""10"",""meeting_duration"":""304"",""name"":""用户P"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655769600&limit=100&order_by=2&start_time=1655683200,"{""PairEndTime"":
""2022-06-21T08:00:00+08:00"", ""PairStartTime"":
""2022-06-20T08:00:00+08:00""}",2022-06-22 06:24:44.399961+00:00
+77,"{""connectionId"":1}","{""id"":""ou_5bfe3a304fc59abf04fae8933d94a918"",""meeting_count"":""18"",""meeting_duration"":""303"",""name"":""用户K"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655769600&limit=100&order_by=2&start_time=1655683200,"{""PairEndTime"":
""2022-06-21T08:00:00+08:00"", ""PairStartTime"":
""2022-06-20T08:00:00+08:00""}",2022-06-22 06:24:44.399961+00:00
+78,"{""connectionId"":1}","{""id"":""ou_e08fc867d0a617e6f681aaa39eb9645a"",""meeting_count"":""9"",""meeting_duration"":""297"",""name"":""用户Q"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655769600&limit=100&order_by=2&start_time=1655683200,"{""PairEndTime"":
""2022-06-21T08:00:00+08:00"", ""PairStartTime"":
""2022-06-20T08:00:00+08:00""}",2022-06-22 06:24:44.399961+00:00
+79,"{""connectionId"":1}","{""id"":""ou_9d3231bd1d159dc68ff0dc601d21a116"",""meeting_count"":""9"",""meeting_duration"":""292"",""name"":""用户R"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655769600&limit=100&order_by=2&start_time=1655683200,"{""PairEndTime"":
""2022-06-21T08:00:00+08:00"", ""PairStartTime"":
""2022-06-20T08:00:00+08:00""}",2022-06-22 06:24:44.399961+00:00
+80,"{""connectionId"":1}","{""id"":""ou_2f60a3ad2dc0eab21c78a3cf89ff3ddd"",""meeting_count"":""5"",""meeting_duration"":""275"",""name"":""用户S"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655769600&limit=100&order_by=2&start_time=1655683200,"{""PairEndTime"":
""2022-06-21T08:00:00+08:00"", ""PairStartTime"":
""2022-06-20T08:00:00+08:00""}",2022-06-22 06:24:44.399961+00:00
+81,"{""connectionId"":1}","{""id"":""ou_133e7e5ff4df17f8484dfc94ff20a1af"",""meeting_count"":""9"",""meeting_duration"":""272"",""name"":""用户T"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655769600&limit=100&order_by=2&start_time=1655683200,"{""PairEndTime"":
""2022-06-21T08:00:00+08:00"", ""PairStartTime"":
""2022-06-20T08:00:00+08:00""}",2022-06-22 06:24:44.399961+00:00
+82,"{""connectionId"":1}","{""id"":""ou_5610f62404011e60446e7fa7ffe4f992"",""meeting_count"":""8"",""meeting_duration"":""265"",""name"":""用户U"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655769600&limit=100&order_by=2&start_time=1655683200,"{""PairEndTime"":
""2022-06-21T08:00:00+08:00"", ""PairStartTime"":
""2022-06-20T08:00:00+08:00""}",2022-06-22 06:24:44.399961+00:00
+83,"{""connectionId"":1}","{""id"":""ou_19c2a0177456027e6c475d4bb948f680"",""meeting_count"":""10"",""meeting_duration"":""262"",""name"":""用户V"",""user_type"":1}",https://open.feishu.cn/open-apis/vc/v1/reports/get_top_user?end_time=1655769600&limit=100&order_by=2&start_time=1655683200,"{""PairEndTime"":
""2022-06-21T08:00:00+08:00"", ""PairStartTime"":
""2022-06-20T08:00:00+08:00""}",2022-06-22 06:24:44.399961+00:00
diff --git
a/plugins/feishu/e2e/snapshot_tables/_tool_feishu_meeting_top_user_items.csv
b/plugins/feishu/e2e/snapshot_tables/_tool_feishu_meeting_top_user_items.csv
new file mode 100644
index 00000000..e71be400
--- /dev/null
+++ b/plugins/feishu/e2e/snapshot_tables/_tool_feishu_meeting_top_user_items.csv
@@ -0,0 +1,29 @@
+connection_id,start_time,name,meeting_count,meeting_duration,user_type,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,2022-06-20T00:00:00.000+00:00,用户A,21,706,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,70,
+1,2022-06-20T00:00:00.000+00:00,用户B,13,417,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,72,
+1,2022-06-20T00:00:00.000+00:00,用户E,17,377,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,74,
+1,2022-06-20T00:00:00.000+00:00,用户H,11,456,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,71,
+1,2022-06-20T00:00:00.000+00:00,用户K,18,303,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,77,
+1,2022-06-20T00:00:00.000+00:00,用户M,12,307,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,75,
+1,2022-06-20T00:00:00.000+00:00,用户O,9,417,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,73,
+1,2022-06-20T00:00:00.000+00:00,用户P,10,304,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,76,
+1,2022-06-20T00:00:00.000+00:00,用户Q,9,297,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,78,
+1,2022-06-20T00:00:00.000+00:00,用户R,9,292,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,79,
+1,2022-06-20T00:00:00.000+00:00,用户S,5,275,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,80,
+1,2022-06-20T00:00:00.000+00:00,用户T,9,272,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,81,
+1,2022-06-20T00:00:00.000+00:00,用户U,8,265,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,82,
+1,2022-06-20T00:00:00.000+00:00,用户V,10,262,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,83,
+1,2022-06-21T00:00:00.000+00:00,用户A,9,256,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,1,
+1,2022-06-21T00:00:00.000+00:00,用户B,7,167,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,2,
+1,2022-06-21T00:00:00.000+00:00,用户C,9,161,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,3,
+1,2022-06-21T00:00:00.000+00:00,用户D,8,151,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,4,
+1,2022-06-21T00:00:00.000+00:00,用户E,11,136,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,5,
+1,2022-06-21T00:00:00.000+00:00,用户F,5,126,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,6,
+1,2022-06-21T00:00:00.000+00:00,用户G,5,110,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,7,
+1,2022-06-21T00:00:00.000+00:00,用户H,3,109,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,8,
+1,2022-06-21T00:00:00.000+00:00,用户I,3,104,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,9,
+1,2022-06-21T00:00:00.000+00:00,用户J,7,102,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,10,
+1,2022-06-21T00:00:00.000+00:00,用户K,6,102,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,11,
+1,2022-06-21T00:00:00.000+00:00,用户L,2,97,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,12,
+1,2022-06-21T00:00:00.000+00:00,用户M,4,97,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,13,
+1,2022-06-21T00:00:00.000+00:00,用户N,2,96,1,"{""connectionId"":1}",_raw_feishu_meeting_top_user_item,14,
diff --git a/plugins/feishu/feishu.go b/plugins/feishu/feishu.go
index 1bce30e9..7ee8b3d3 100644
--- a/plugins/feishu/feishu.go
+++ b/plugins/feishu/feishu.go
@@ -18,79 +18,25 @@ limitations under the License.
package main
import (
- "github.com/apache/incubator-devlake/migration"
- "github.com/apache/incubator-devlake/plugins/core"
-
"github.com/apache/incubator-devlake/plugins/feishu/models/migrationscripts"
- "github.com/apache/incubator-devlake/plugins/feishu/tasks"
+ "github.com/apache/incubator-devlake/plugins/feishu/impl"
"github.com/apache/incubator-devlake/runner"
- "github.com/mitchellh/mapstructure"
"github.com/spf13/cobra"
- "github.com/spf13/viper"
- "gorm.io/gorm"
)
-var _ core.PluginMeta = (*Feishu)(nil)
-var _ core.PluginInit = (*Feishu)(nil)
-var _ core.PluginTask = (*Feishu)(nil)
-var _ core.PluginApi = (*Feishu)(nil)
-var _ core.Migratable = (*Feishu)(nil)
-
-type Feishu struct{}
-
-func (plugin Feishu) Init(config *viper.Viper, logger core.Logger, db
*gorm.DB) error {
- return nil
-}
-
-func (plugin Feishu) Description() string {
- return "To collect and enrich data from Feishu"
-}
-
-func (plugin Feishu) SubTaskMetas() []core.SubTaskMeta {
- return []core.SubTaskMeta{
- tasks.CollectMeetingTopUserItemMeta,
- tasks.ExtractMeetingTopUserItemMeta,
- }
-}
-
-func (plugin Feishu) PrepareTaskData(taskCtx core.TaskContext, options
map[string]interface{}) (interface{}, error) {
- var op tasks.FeishuOptions
- err := mapstructure.Decode(options, &op)
- if err != nil {
- return nil, err
- }
- apiClient, err := tasks.NewFeishuApiClient(taskCtx)
- if err != nil {
- return nil, err
- }
- return &tasks.FeishuTaskData{
- Options: &op,
- ApiClient: apiClient,
- }, nil
-}
-
-func (plugin Feishu) RootPkgPath() string {
- return "github.com/apache/incubator-devlake/plugins/feishu"
-}
-
-func (plugin Feishu) MigrationScripts() []migration.Script {
- return migrationscripts.All()
-}
-
-func (plugin Feishu) ApiResources()
map[string]map[string]core.ApiResourceHandler {
- return map[string]map[string]core.ApiResourceHandler{}
-}
-
-var PluginEntry Feishu
+var PluginEntry impl.Feishu
// standalone mode for debugging
func main() {
- feishuCmd := &cobra.Command{Use: "feishu"}
- numOfDaysToCollect := feishuCmd.Flags().IntP("numOfDaysToCollect", "n",
8, "feishu collect days")
- _ = feishuCmd.MarkFlagRequired("numOfDaysToCollect")
- feishuCmd.Run = func(cmd *cobra.Command, args []string) {
+ cmd := &cobra.Command{Use: "feishu"}
+ connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "feishu
connection id")
+ numOfDaysToCollect := cmd.Flags().IntP("numOfDaysToCollect", "n", 8,
"feishu collect days")
+ _ = cmd.MarkFlagRequired("connectionId")
+ _ = cmd.MarkFlagRequired("numOfDaysToCollect")
+ cmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
+ "connectionId": *connectionId,
"numOfDaysToCollect": *numOfDaysToCollect,
})
}
- runner.RunCmd(feishuCmd)
+ runner.RunCmd(cmd)
}
diff --git a/plugins/feishu/feishu.go b/plugins/feishu/impl/impl.go
similarity index 62%
copy from plugins/feishu/feishu.go
copy to plugins/feishu/impl/impl.go
index 1bce30e9..cc4a4e03 100644
--- a/plugins/feishu/feishu.go
+++ b/plugins/feishu/impl/impl.go
@@ -15,18 +15,20 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package main
+package impl
import (
+ "github.com/mitchellh/mapstructure"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
+
"github.com/apache/incubator-devlake/migration"
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/feishu/api"
+ "github.com/apache/incubator-devlake/plugins/feishu/models"
"github.com/apache/incubator-devlake/plugins/feishu/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/feishu/tasks"
- "github.com/apache/incubator-devlake/runner"
- "github.com/mitchellh/mapstructure"
- "github.com/spf13/cobra"
- "github.com/spf13/viper"
- "gorm.io/gorm"
+ "github.com/apache/incubator-devlake/plugins/helper"
)
var _ core.PluginMeta = (*Feishu)(nil)
@@ -38,6 +40,36 @@ var _ core.Migratable = (*Feishu)(nil)
type Feishu struct{}
func (plugin Feishu) Init(config *viper.Viper, logger core.Logger, db
*gorm.DB) error {
+ api.Init(config, logger, db)
+
+ // FIXME after config-ui support feishu plugin
+ // save env to db where name=feishu
+ connection := &models.FeishuConnection{}
+ if db.Migrator().HasTable(connection) {
+ err := db.Find(connection, map[string]string{"name":
"Feishu"}).Error
+ if err != nil {
+ return err
+ }
+ if connection.ID != 0 {
+ encodeKey := config.GetString(core.EncodeKeyEnvStr)
+ connection.Endpoint =
config.GetString(`FEISHU_ENDPOINT`)
+ connection.AppId = config.GetString(`FEISHU_APPID`)
+ connection.SecretKey =
config.GetString(`FEISHU_APPSCRECT`)
+ if connection.Endpoint != `` && connection.AppId != ``
&& connection.SecretKey != `` && encodeKey != `` {
+ err = helper.UpdateEncryptFields(connection,
func(plaintext string) (string, error) {
+ return core.Encrypt(encodeKey,
plaintext)
+ })
+ if err != nil {
+ return err
+ }
+ // update from .env and save to db
+ err = db.Updates(connection).Error
+ if err != nil {
+ return err
+ }
+ }
+ }
+ }
return nil
}
@@ -58,7 +90,18 @@ func (plugin Feishu) PrepareTaskData(taskCtx
core.TaskContext, options map[strin
if err != nil {
return nil, err
}
- apiClient, err := tasks.NewFeishuApiClient(taskCtx)
+
+ connectionHelper := helper.NewConnectionHelper(
+ taskCtx,
+ nil,
+ )
+ connection := &models.FeishuConnection{}
+ err = connectionHelper.FirstById(connection, op.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+
+ apiClient, err := tasks.NewFeishuApiClient(taskCtx, connection)
if err != nil {
return nil, err
}
@@ -79,18 +122,3 @@ func (plugin Feishu) MigrationScripts() []migration.Script {
func (plugin Feishu) ApiResources()
map[string]map[string]core.ApiResourceHandler {
return map[string]map[string]core.ApiResourceHandler{}
}
-
-var PluginEntry Feishu
-
-// standalone mode for debugging
-func main() {
- feishuCmd := &cobra.Command{Use: "feishu"}
- numOfDaysToCollect := feishuCmd.Flags().IntP("numOfDaysToCollect", "n",
8, "feishu collect days")
- _ = feishuCmd.MarkFlagRequired("numOfDaysToCollect")
- feishuCmd.Run = func(cmd *cobra.Command, args []string) {
- runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- "numOfDaysToCollect": *numOfDaysToCollect,
- })
- }
- runner.RunCmd(feishuCmd)
-}
diff --git a/plugins/feishu/models/meeting_top_user_item.go
b/plugins/feishu/models/connection.go
similarity index 58%
copy from plugins/feishu/models/meeting_top_user_item.go
copy to plugins/feishu/models/connection.go
index 9b50c0c0..b721e7af 100644
--- a/plugins/feishu/models/meeting_top_user_item.go
+++ b/plugins/feishu/models/connection.go
@@ -17,20 +17,20 @@ limitations under the License.
package models
-import (
- "github.com/apache/incubator-devlake/models/common"
- "time"
-)
+import "github.com/apache/incubator-devlake/plugins/helper"
-type FeishuMeetingTopUserItem struct {
- common.NoPKModel `json:"-"`
- StartTime time.Time `gorm:"primaryKey"`
- Name string `json:"name"
gorm:"primaryKey;type:varchar(255)"`
- MeetingCount string `json:"meeting_count"
gorm:"type:varchar(255)"`
- MeetingDuration string `json:"meeting_duration"
gorm:"type:varchar(255)"`
- UserType int64 `json:"user_type"`
+type TestConnectionRequest struct {
+ Endpoint string `json:"endpoint" validate:"required,url"`
+ AppId string `mapstructure:"app_id" validate:"required"
json:"app_id"`
+ SecretKey string `mapstructure:"secret_key" validate:"required"
json:"secret_key"`
+ Proxy string `json:"proxy"`
}
-func (FeishuMeetingTopUserItem) TableName() string {
- return "_tool_feishu_meeting_top_user_items"
+type FeishuConnection struct {
+ helper.RestConnection `mapstructure:",squash"`
+ helper.AppKey `mapstructure:",squash"`
+}
+
+func (FeishuConnection) TableName() string {
+ return "_tool_feishu_connections"
}
diff --git a/plugins/feishu/models/meeting_top_user_item.go
b/plugins/feishu/models/meeting_top_user_item.go
index 9b50c0c0..01625590 100644
--- a/plugins/feishu/models/meeting_top_user_item.go
+++ b/plugins/feishu/models/meeting_top_user_item.go
@@ -24,6 +24,7 @@ import (
type FeishuMeetingTopUserItem struct {
common.NoPKModel `json:"-"`
+ ConnectionId uint64 `gorm:"primaryKey"`
StartTime time.Time `gorm:"primaryKey"`
Name string `json:"name"
gorm:"primaryKey;type:varchar(255)"`
MeetingCount string `json:"meeting_count"
gorm:"type:varchar(255)"`
diff --git a/plugins/feishu/models/migrationscripts/init_schema.go
b/plugins/feishu/models/migrationscripts/archived/connection.go
similarity index 53%
copy from plugins/feishu/models/migrationscripts/init_schema.go
copy to plugins/feishu/models/migrationscripts/archived/connection.go
index 6f6cd06a..8e8e695f 100644
--- a/plugins/feishu/models/migrationscripts/init_schema.go
+++ b/plugins/feishu/models/migrationscripts/archived/connection.go
@@ -15,27 +15,22 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package archived
import (
- "context"
-
-
"github.com/apache/incubator-devlake/plugins/feishu/models/migrationscripts/archived"
- "gorm.io/gorm"
+ commonArchived
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
)
-type InitSchemas struct{}
-
-func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
- return db.Migrator().AutoMigrate(
- &archived.FeishuMeetingTopUserItem{},
- )
-}
-
-func (*InitSchemas) Version() uint64 {
- return 20220407201134
+type FeishuConnection struct {
+ commonArchived.Model
+ Name string `gorm:"type:varchar(100);uniqueIndex" json:"name"
validate:"required"`
+ Endpoint string `mapstructure:"endpoint" env:"GITHUB_ENDPOINT"
validate:"required"`
+ Proxy string `mapstructure:"proxy" env:"GITHUB_PROXY"`
+ RateLimit int `comment:"api request rate limit per hour"`
+ AppId string `mapstructure:"app_id" validate:"required"
json:"app_id"`
+ SecretKey string `mapstructure:"secret_key" validate:"required"
json:"secret_key" encrypt:"yes"`
}
-func (*InitSchemas) Name() string {
- return "Feishu init schemas"
+func (FeishuConnection) TableName() string {
+ return "_tool_feishu_connections"
}
diff --git
a/plugins/feishu/models/migrationscripts/archived/meeting_top_user_item.go
b/plugins/feishu/models/migrationscripts/archived/meeting_top_user_item.go
index 149a6da9..733f9897 100644
--- a/plugins/feishu/models/migrationscripts/archived/meeting_top_user_item.go
+++ b/plugins/feishu/models/migrationscripts/archived/meeting_top_user_item.go
@@ -18,19 +18,18 @@ limitations under the License.
package archived
import (
+ "github.com/apache/incubator-devlake/models/common"
"time"
-
- "github.com/apache/incubator-devlake/models/migrationscripts/archived"
)
type FeishuMeetingTopUserItem struct {
- archived.Model `json:"-"`
- StartTime time.Time
- MeetingCount string `json:"meeting_count" gorm:"type:varchar(255)"`
- MeetingDuration string `json:"meeting_duration"
gorm:"type:varchar(255)"`
- Name string `json:"name" gorm:"type:varchar(255)"`
- UserType int64 `json:"user_type"`
- archived.RawDataOrigin
+ common.NoPKModel `json:"-"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ StartTime time.Time `gorm:"primaryKey"`
+ Name string `json:"name"
gorm:"primaryKey;type:varchar(255)"`
+ MeetingCount string `json:"meeting_count"
gorm:"type:varchar(255)"`
+ MeetingDuration string `json:"meeting_duration"
gorm:"type:varchar(255)"`
+ UserType int64 `json:"user_type"`
}
func (FeishuMeetingTopUserItem) TableName() string {
diff --git a/plugins/feishu/models/migrationscripts/init_schema.go
b/plugins/feishu/models/migrationscripts/init_schema.go
index 6f6cd06a..020ff6d8 100644
--- a/plugins/feishu/models/migrationscripts/init_schema.go
+++ b/plugins/feishu/models/migrationscripts/init_schema.go
@@ -19,21 +19,58 @@ package migrationscripts
import (
"context"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/feishu/models/migrationscripts/archived"
"gorm.io/gorm"
)
-type InitSchemas struct{}
+type InitSchemas struct {
+ config core.ConfigGetter
+}
+
+func (u *InitSchemas) SetConfigGetter(config core.ConfigGetter) {
+ u.config = config
+}
-func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
- return db.Migrator().AutoMigrate(
+func (u *InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
+ err := db.Migrator().DropTable(
+ &archived.FeishuConnection{},
&archived.FeishuMeetingTopUserItem{},
)
+ if err != nil {
+ return err
+ }
+ err = db.Migrator().CreateTable(
+ &archived.FeishuConnection{},
+ &archived.FeishuMeetingTopUserItem{},
+ )
+ if err != nil {
+ return err
+ }
+
+ encodeKey := u.config.GetString(core.EncodeKeyEnvStr)
+ connection := &archived.FeishuConnection{}
+ connection.Endpoint = u.config.GetString(`FEISHU_ENDPOINT`)
+ connection.AppId = u.config.GetString(`FEISHU_APPID`)
+ connection.SecretKey = u.config.GetString(`FEISHU_APPSCRECT`)
+ connection.Name = `Feishu`
+ if connection.Endpoint != `` && connection.AppId != `` &&
connection.SecretKey != `` && encodeKey != `` {
+ err = helper.UpdateEncryptFields(connection, func(plaintext
string) (string, error) {
+ return core.Encrypt(encodeKey, plaintext)
+ })
+ if err != nil {
+ return err
+ }
+ // update from .env and save to db
+ db.Create(connection)
+ }
+ return nil
}
func (*InitSchemas) Version() uint64 {
- return 20220407201134
+ return 20220620000001
}
func (*InitSchemas) Name() string {
diff --git a/plugins/feishu/models/migrationscripts/register.go
b/plugins/feishu/models/migrationscripts/register.go
index 57d8ea48..06d924d0 100644
--- a/plugins/feishu/models/migrationscripts/register.go
+++ b/plugins/feishu/models/migrationscripts/register.go
@@ -24,6 +24,6 @@ import (
// All return all the migration scripts
func All() []migration.Script {
return []migration.Script{
- new(InitSchemas), new(UpdateSchemas20220524),
+ new(InitSchemas),
}
}
diff --git a/plugins/feishu/models/migrationscripts/updateSchemas20220526.go
b/plugins/feishu/models/migrationscripts/updateSchemas20220526.go
deleted file mode 100644
index 634046d4..00000000
--- a/plugins/feishu/models/migrationscripts/updateSchemas20220526.go
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package migrationscripts
-
-import (
- "context"
- "github.com/apache/incubator-devlake/models/common"
-
"github.com/apache/incubator-devlake/plugins/feishu/models/migrationscripts/archived"
- "gorm.io/gorm/clause"
- "time"
-
- "gorm.io/gorm"
-)
-
-type FeishuMeetingTopUserItem20220524Temp struct {
- common.NoPKModel `json:"-"`
- StartTime time.Time `gorm:"primaryKey"`
- Name string `json:"name"
gorm:"primaryKey;type:varchar(255)"`
- MeetingCount string `json:"meeting_count"
gorm:"type:varchar(255)"`
- MeetingDuration string `json:"meeting_duration"
gorm:"type:varchar(255)"`
- UserType int64 `json:"user_type"`
-}
-
-func (FeishuMeetingTopUserItem20220524Temp) TableName() string {
- return "_tool_feishu_meeting_top_user_items_tmp"
-}
-
-type FeishuMeetingTopUserItem20220524 struct {
-}
-
-func (FeishuMeetingTopUserItem20220524) TableName() string {
- return "_tool_feishu_meeting_top_user_items"
-}
-
-type UpdateSchemas20220524 struct{}
-
-func (*UpdateSchemas20220524) Up(ctx context.Context, db *gorm.DB) error {
- cursor, err := db.Model(archived.FeishuMeetingTopUserItem{}).Rows()
- if err != nil {
- return err
- }
- defer cursor.Close()
- // 1. create a temporary table to store unique records
- err = db.Migrator().CreateTable(FeishuMeetingTopUserItem20220524Temp{})
- if err != nil {
- return err
- }
- // 2. dedupe records and insert into the temporary table
- for cursor.Next() {
- inputRow := FeishuMeetingTopUserItem20220524Temp{}
- err := db.ScanRows(cursor, &inputRow)
- if err != nil {
- return err
- }
- err = db.Clauses(clause.OnConflict{UpdateAll:
true}).Create(inputRow).Error
- if err != nil {
- return err
- }
- }
- // 3. drop old table
- err = db.Migrator().DropTable(archived.FeishuMeetingTopUserItem{})
- if err != nil {
- return err
- }
- // 4. rename the temporary table to the old table
- return
db.Migrator().RenameTable(FeishuMeetingTopUserItem20220524Temp{},
FeishuMeetingTopUserItem20220524{})
-}
-
-func (*UpdateSchemas20220524) Version() uint64 {
- return 20220524000001
-}
-
-func (*UpdateSchemas20220524) Name() string {
- return "change primary column `id` to start_time+name"
-}
diff --git a/plugins/feishu/tasks/api_client.go
b/plugins/feishu/tasks/api_client.go
index da54b5f9..941403a4 100644
--- a/plugins/feishu/tasks/api_client.go
+++ b/plugins/feishu/tasks/api_client.go
@@ -19,60 +19,33 @@ package tasks
import (
"fmt"
+ "github.com/apache/incubator-devlake/plugins/feishu/apimodels"
+ "github.com/apache/incubator-devlake/plugins/feishu/models"
"net/http"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
- "github.com/apache/incubator-devlake/utils"
)
-type ApiAccessTokenRequest struct {
- AppId string `json:"app_id"`
- AppSecret string `json:"app_secret"`
-}
-
-type ApiAccessTokenResponse struct {
- Code int `json:"code"`
- Msg string `json:"msg"`
- AppAccessToken string `json:"app_access_token"`
- TenantAccessToken string `json:"tenant_access_token"`
- Expire int `json:"expire"`
-}
-
const AUTH_ENDPOINT = "https://open.feishu.cn";
const ENDPOINT = "https://open.feishu.cn/open-apis/vc/v1";
-func NewFeishuApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient,
error) {
- // load and process cconfiguration
- appId := taskCtx.GetConfig("FEISHU_APPID")
- if appId == "" {
- return nil, fmt.Errorf("invalid FEISHU_APPID")
- }
- secretKey := taskCtx.GetConfig("FEISHU_APPSCRECT")
- if secretKey == "" {
- return nil, fmt.Errorf("invalid FEISHU_APPSCRECT")
- }
- userRateLimit, err :=
utils.StrToIntOr(taskCtx.GetConfig("FEISHU_API_REQUESTS_PER_HOUR"), 18000)
- if err != nil {
- return nil, err
- }
- proxy := taskCtx.GetConfig("FEISHU_PROXY")
-
- authApiClient, err := helper.NewApiClient(AUTH_ENDPOINT, nil, 0, proxy,
taskCtx.GetContext())
+func NewFeishuApiClient(taskCtx core.TaskContext, connection
*models.FeishuConnection) (*helper.ApiAsyncClient, error) {
+ authApiClient, err := helper.NewApiClient(AUTH_ENDPOINT, nil, 0,
connection.Proxy, taskCtx.GetContext())
if err != nil {
return nil, err
}
// request for access token
- tokenReqBody := &ApiAccessTokenRequest{
- AppId: appId,
- AppSecret: secretKey,
+ tokenReqBody := &apimodels.ApiAccessTokenRequest{
+ AppId: connection.AppId,
+ AppSecret: connection.SecretKey,
}
tokenRes, err :=
authApiClient.Post("open-apis/auth/v3/tenant_access_token/internal", nil,
tokenReqBody, nil)
if err != nil {
return nil, err
}
- tokenResBody := &ApiAccessTokenResponse{}
+ tokenResBody := &apimodels.ApiAccessTokenResponse{}
err = helper.UnmarshalResponse(tokenRes, tokenResBody)
if err != nil {
return nil, err
@@ -81,7 +54,7 @@ func NewFeishuApiClient(taskCtx core.TaskContext)
(*helper.ApiAsyncClient, error
return nil, fmt.Errorf("failed to request access token")
}
// real request apiClient
- apiClient, err := helper.NewApiClient(ENDPOINT, nil, 0, proxy,
taskCtx.GetContext())
+ apiClient, err := helper.NewApiClient(ENDPOINT, nil, 0,
connection.Proxy, taskCtx.GetContext())
if err != nil {
return nil, err
}
@@ -99,7 +72,7 @@ func NewFeishuApiClient(taskCtx core.TaskContext)
(*helper.ApiAsyncClient, error
// create async api client
asyncApiCLient, err := helper.CreateAsyncApiClient(taskCtx, apiClient,
&helper.ApiRateLimitCalculator{
- UserRateLimitPerHour: userRateLimit,
+ UserRateLimitPerHour: connection.RateLimit,
})
if err != nil {
return nil, err
diff --git a/plugins/feishu/tasks/meeting_top_user_item_collector.go
b/plugins/feishu/tasks/meeting_top_user_item_collector.go
index 4b8ce214..977a9eb5 100644
--- a/plugins/feishu/tasks/meeting_top_user_item_collector.go
+++ b/plugins/feishu/tasks/meeting_top_user_item_collector.go
@@ -40,18 +40,17 @@ func CollectMeetingTopUserItem(taskCtx core.SubTaskContext)
error {
if err != nil {
return err
}
- incremental := false
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: FeishuApiParams{
- ApiResName: "top_user_report",
+ ConnectionId: data.Options.ConnectionId,
},
Table: RAW_MEETING_TOP_USER_ITEM_TABLE,
},
ApiClient: data.ApiClient,
- Incremental: incremental,
+ Incremental: false,
Input: iterator,
UrlTemplate: "/reports/get_top_user",
Query: func(reqData *helper.RequestData) (url.Values, error) {
diff --git a/plugins/feishu/tasks/meeting_top_user_item_extractor.go
b/plugins/feishu/tasks/meeting_top_user_item_extractor.go
index fd4c3ca5..6a704393 100644
--- a/plugins/feishu/tasks/meeting_top_user_item_extractor.go
+++ b/plugins/feishu/tasks/meeting_top_user_item_extractor.go
@@ -27,11 +27,12 @@ import (
var _ core.SubTaskEntryPoint = ExtractMeetingTopUserItem
func ExtractMeetingTopUserItem(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*FeishuTaskData)
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: FeishuApiParams{
- ApiResName: "top_user_report",
+ ConnectionId: data.Options.ConnectionId,
},
Table: RAW_MEETING_TOP_USER_ITEM_TABLE,
},
@@ -48,7 +49,8 @@ func ExtractMeetingTopUserItem(taskCtx core.SubTaskContext)
error {
}
results := make([]interface{}, 0)
results = append(results,
&models.FeishuMeetingTopUserItem{
- StartTime:
rawInput.PairStartTime.AddDate(0, 0, -1),
+ ConnectionId: data.Options.ConnectionId,
+ StartTime: rawInput.PairStartTime,
MeetingCount: body.MeetingCount,
MeetingDuration: body.MeetingDuration,
Name: body.Name,
diff --git a/plugins/feishu/tasks/task_data.go
b/plugins/feishu/tasks/task_data.go
index aad0549d..a5d461db 100644
--- a/plugins/feishu/tasks/task_data.go
+++ b/plugins/feishu/tasks/task_data.go
@@ -23,12 +23,12 @@ import (
)
type FeishuApiParams struct {
- ApiResName string `json:"apiResName"`
+ ConnectionId uint64 `json:"connectionId"`
}
type FeishuOptions struct {
- NumOfDaysToCollect float64 `json:"numOfDaysToCollect"`
- Tasks []string `json:"tasks,omitempty"`
+ ConnectionId uint64 `json:"connectionId"`
+ NumOfDaysToCollect float64 `json:"numOfDaysToCollect"`
}
type FeishuTaskData struct {
diff --git a/plugins/tapd/e2e/snapshot_tables/issue_commits_story.csv
b/plugins/tapd/e2e/snapshot_tables/issue_commits_story.csv
new file mode 100644
index 00000000..9d9ded75
--- /dev/null
+++ b/plugins/tapd/e2e/snapshot_tables/issue_commits_story.csv
@@ -0,0 +1,2 @@
+issue_id,commit_sha,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+tapd:TapdIssue:1:11991001049945,testcommit,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_story_commits,1,
