This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 47ebe5d54 feat: collect slack channel/message/thread (#5016)
47ebe5d54 is described below
commit 47ebe5d547bdab26e6700c9f1a927c5a58d2c193
Author: Likyh <[email protected]>
AuthorDate: Fri May 12 08:57:01 2023 +0800
feat: collect slack channel/message/thread (#5016)
* feat: collect slack channel/message/thread
* fix: fix collect bug when use GetNextPageCustomData
---
backend/helpers/pluginhelper/api/api_collector.go | 3 +-
backend/plugins/slack/README.md | 1 +
backend/plugins/slack/api/connection.go | 146 ++++++++++++++++++
backend/plugins/slack/api/init.go | 37 +++++
backend/plugins/slack/api/swagger.go | 44 ++++++
backend/plugins/slack/apimodels/access_token.go | 31 ++++
backend/plugins/slack/apimodels/im_result.go | 166 +++++++++++++++++++++
backend/plugins/slack/impl/impl.go | 130 ++++++++++++++++
backend/plugins/slack/models/channel.go | 52 +++++++
backend/plugins/slack/models/channel_message.go | 46 ++++++
backend/plugins/slack/models/connection.go | 38 +++++
.../migrationscripts/20230421_add_init_tables.go | 78 ++++++++++
.../models/migrationscripts/archived/channel.go | 52 +++++++
.../migrationscripts/archived/channel_message.go | 46 ++++++
.../models/migrationscripts/archived/connection.go | 35 +++++
.../slack/models/migrationscripts/register.go | 29 ++++
backend/plugins/slack/slack.go | 39 +++++
backend/plugins/slack/tasks/api_client.go | 42 ++++++
backend/plugins/slack/tasks/channel_collector.go | 91 +++++++++++
backend/plugins/slack/tasks/channel_extractor.go | 62 ++++++++
.../slack/tasks/channel_message_collector.go | 118 +++++++++++++++
.../slack/tasks/channel_message_extractor.go | 85 +++++++++++
backend/plugins/slack/tasks/task_data.go | 35 +++++
backend/plugins/slack/tasks/thread_collector.go | 121 +++++++++++++++
backend/plugins/slack/tasks/thread_extractor.go | 85 +++++++++++
25 files changed, 1611 insertions(+), 1 deletion(-)
diff --git a/backend/helpers/pluginhelper/api/api_collector.go
b/backend/helpers/pluginhelper/api/api_collector.go
index 845658bef..517025dea 100644
--- a/backend/helpers/pluginhelper/api/api_collector.go
+++ b/backend/helpers/pluginhelper/api/api_collector.go
@@ -265,7 +265,8 @@ func (collector *ApiCollector)
fetchPagesSequentially(reqData *RequestData) {
reqData.CustomData = customData
reqData.Pager.Skip += collector.args.PageSize
reqData.Pager.Page += 1
- return collect()
+ collector.args.ApiClient.NextTick(collect)
+ return nil
})
return nil
}
diff --git a/backend/plugins/slack/README.md b/backend/plugins/slack/README.md
new file mode 100644
index 000000000..a792324f9
--- /dev/null
+++ b/backend/plugins/slack/README.md
@@ -0,0 +1 @@
+Please see details in the [Apache DevLake
website](https://devlake.apache.org/docs/Plugins/slack)
\ No newline at end of file
diff --git a/backend/plugins/slack/api/connection.go
b/backend/plugins/slack/api/connection.go
new file mode 100644
index 000000000..b1bf7ccd3
--- /dev/null
+++ b/backend/plugins/slack/api/connection.go
@@ -0,0 +1,146 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "github.com/apache/incubator-devlake/server/api/shared"
+ "net/http"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/slack/models"
+)
+
+type SlackTestConnResponse struct {
+ shared.ApiBody
+ Connection *models.SlackConn
+}
+
+// @Summary test slack connection
+// @Description Test slack Connection. endpoint:
https://open.slack.cn/open-apis/
+// @Tags plugins/slack
+// @Param body body models.SlackConn true "json body"
+// @Success 200 {object} SlackTestConnResponse "Success"
+// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 500 {string} errcode.Error "Internal Error"
+// @Router /plugins/slack/test [POST]
+func TestConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ // process input
+ var connection models.SlackConn
+ if err := api.Decode(input.Body, &connection, vld); err != nil {
+ return nil, errors.BadInput.Wrap(err, "could not decode request
parameters")
+ }
+
+ // test connection
+ _, err := api.NewApiClientFromConnection(context.TODO(), basicRes,
&connection)
+
+ body := SlackTestConnResponse{}
+ body.Success = true
+ body.Message = "success"
+ body.Connection = &connection
+ if err != nil {
+ return nil, err
+ }
+ return &plugin.ApiResourceOutput{Body: body, Status: 200}, nil
+}
+
+// @Summary create slack connection
+// @Description Create slack connection
+// @Tags plugins/slack
+// @Param body body models.SlackConnection true "json body"
+// @Success 200 {object} models.SlackConnection
+// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 500 {string} errcode.Error "Internal Error"
+// @Router /plugins/slack/connections [POST]
+func PostConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ connection := &models.SlackConnection{}
+ err := connectionHelper.Create(connection, input)
+ if err != nil {
+ return nil, err
+ }
+ return &plugin.ApiResourceOutput{Body: connection, Status:
http.StatusOK}, nil
+}
+
+// @Summary patch slack connection
+// @Description Patch slack connection
+// @Tags plugins/slack
+// @Param body body models.SlackConnection true "json body"
+// @Success 200 {object} models.SlackConnection
+// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 500 {string} errcode.Error "Internal Error"
+// @Router /plugins/slack/connections/{connectionId} [PATCH]
+func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ connection := &models.SlackConnection{}
+ err := connectionHelper.Patch(connection, input)
+ if err != nil {
+ return nil, err
+ }
+ return &plugin.ApiResourceOutput{Body: connection, Status:
http.StatusOK}, nil
+}
+
+// @Summary delete a slack connection
+// @Description Delete a slack connection
+// @Tags plugins/slack
+// @Success 200 {object} models.SlackConnection
+// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 500 {string} errcode.Error "Internal Error"
+// @Router /plugins/slack/connections/{connectionId} [DELETE]
+func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ connection := &models.SlackConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+ err = connectionHelper.Delete(connection)
+ return &plugin.ApiResourceOutput{Body: connection}, err
+}
+
+// @Summary get all slack connections
+// @Description Get all slack connections
+// @Tags plugins/slack
+// @Success 200 {object} models.SlackConnection
+// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 500 {string} errcode.Error "Internal Error"
+// @Router /plugins/slack/connections [GET]
+func ListConnections(_ *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ var connections []models.SlackConnection
+ err := connectionHelper.List(&connections)
+ if err != nil {
+ return nil, err
+ }
+
+ return &plugin.ApiResourceOutput{Body: connections}, nil
+}
+
+// @Summary get slack connection detail
+// @Description Get slack connection detail
+// @Tags plugins/slack
+// @Success 200 {object} models.SlackConnection
+// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 500 {string} errcode.Error "Internal Error"
+// @Router /plugins/slack/connections/{connectionId} [GET]
+func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ connection := &models.SlackConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+ return &plugin.ApiResourceOutput{Body: connection}, err
+}
diff --git a/backend/plugins/slack/api/init.go
b/backend/plugins/slack/api/init.go
new file mode 100644
index 000000000..d92c2b334
--- /dev/null
+++ b/backend/plugins/slack/api/init.go
@@ -0,0 +1,37 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/go-playground/validator/v10"
+)
+
+var vld *validator.Validate
+var connectionHelper *api.ConnectionApiHelper
+var basicRes context.BasicRes
+
+func Init(br context.BasicRes) {
+ basicRes = br
+ vld = validator.New()
+ connectionHelper = api.NewConnectionHelper(
+ basicRes,
+ vld,
+ )
+}
diff --git a/backend/plugins/slack/api/swagger.go
b/backend/plugins/slack/api/swagger.go
new file mode 100644
index 000000000..25e67ba44
--- /dev/null
+++ b/backend/plugins/slack/api/swagger.go
@@ -0,0 +1,44 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+// @Summary blueprints plan for slack
+// @Description blueprints plan for slack
+// @Tags plugins/slack
+// @Accept application/json
+// @Param blueprint body SlackBlueprintPlan true "json"
+// @Router /blueprints/slack/blueprint-plan [post]
+func _() {}
+
+type SlackBlueprintPlan [][]struct {
+ Plugin string `json:"plugin"`
+ Options struct{} `json:"options"`
+}
+
+// @Summary pipelines plan for slack
+// @Description pipelines plan for slack
+// @Tags plugins/slack
+// @Accept application/json
+// @Param pipeline body SlackPipelinePlan true "json"
+// @Router /pipelines/slack/pipeline-plan [post]
+func _() {}
+
+type SlackPipelinePlan [][]struct {
+ Plugin string `json:"plugin"`
+ Options struct{} `json:"options"`
+}
diff --git a/backend/plugins/slack/apimodels/access_token.go
b/backend/plugins/slack/apimodels/access_token.go
new file mode 100644
index 000000000..d61f2bf92
--- /dev/null
+++ b/backend/plugins/slack/apimodels/access_token.go
@@ -0,0 +1,31 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apimodels
+
+type ApiAccessTokenRequest struct {
+ AppId string `json:"app_id"`
+ AppSecret string `json:"app_secret"`
+}
+
+type ApiAccessTokenResponse struct {
+ Code int `json:"code"`
+ Msg string `json:"msg"`
+ AppAccessToken string `json:"app_access_token"`
+ TenantAccessToken string `json:"tenant_access_token"`
+ Expire int `json:"expire"`
+}
diff --git a/backend/plugins/slack/apimodels/im_result.go
b/backend/plugins/slack/apimodels/im_result.go
new file mode 100644
index 000000000..f1b2cde72
--- /dev/null
+++ b/backend/plugins/slack/apimodels/im_result.go
@@ -0,0 +1,166 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apimodels
+
+import "encoding/json"
+
+type SlackChannelApiResult struct {
+ Ok bool `json:"ok"`
+ Channels []json.RawMessage `json:"channels"`
+ ResponseMetadata struct {
+ NextCursor string `json:"next_cursor"`
+ } `json:"response_metadata"`
+}
+
+type SlackChannelMessageApiResult struct {
+ Ok bool `json:"ok"`
+ Messages []json.RawMessage `json:"messages"`
+ ResponseMetadata struct {
+ NextCursor string `json:"next_cursor"`
+ } `json:"response_metadata"`
+}
+
+type SlackChannelMessageResultItem struct {
+ ClientMsgId string `json:"client_msg_id"`
+ Type string `json:"type"`
+ Subtype string `json:"subtype"`
+ Ts string `json:"ts"`
+ ThreadTs string `json:"thread_ts"`
+ User string `json:"user"`
+ Text string `json:"text"`
+
+ Team string `json:"team"`
+ ReplyCount int `json:"reply_count"`
+ ReplyUsersCount int `json:"reply_users_count"`
+ LatestReply string `json:"latest_reply"`
+ ReplyUsers []string `json:"reply_users"`
+ IsLocked bool `json:"is_locked"`
+ Subscribed bool `json:"subscribed"`
+ ParentUserId string `json:"parent_user_id"`
+
+ Files []struct {
+ Id string `json:"id"`
+ Created int `json:"created"`
+ Timestamp int `json:"timestamp"`
+ Name string `json:"name"`
+ Title string `json:"title"`
+ Mimetype string `json:"mimetype"`
+ Filetype string `json:"filetype"`
+ PrettyType string `json:"pretty_type"`
+ User string `json:"user"`
+ UserTeam string `json:"user_team"`
+ Editable bool `json:"editable"`
+ Size int `json:"size"`
+ Mode string `json:"mode"`
+ IsExternal bool `json:"is_external"`
+ ExternalType string `json:"external_type"`
+ IsPublic bool `json:"is_public"`
+ PublicUrlShared bool `json:"public_url_shared"`
+ DisplayAsBot bool `json:"display_as_bot"`
+ Username string `json:"username"`
+ UrlPrivate string `json:"url_private"`
+ UrlPrivateDownload string `json:"url_private_download"`
+ MediaDisplayType string `json:"media_display_type"`
+ Thumb64 string `json:"thumb_64"`
+ Thumb80 string `json:"thumb_80"`
+ Thumb360 string `json:"thumb_360"`
+ Thumb360W int `json:"thumb_360_w"`
+ Thumb360H int `json:"thumb_360_h"`
+ Thumb480 string `json:"thumb_480"`
+ Thumb480W int `json:"thumb_480_w"`
+ Thumb480H int `json:"thumb_480_h"`
+ Thumb160 string `json:"thumb_160"`
+ Thumb720 string `json:"thumb_720"`
+ Thumb720W int `json:"thumb_720_w"`
+ Thumb720H int `json:"thumb_720_h"`
+ Thumb800 string `json:"thumb_800"`
+ Thumb800W int `json:"thumb_800_w"`
+ Thumb800H int `json:"thumb_800_h"`
+ Thumb960 string `json:"thumb_960"`
+ Thumb960W int `json:"thumb_960_w"`
+ Thumb960H int `json:"thumb_960_h"`
+ Thumb1024 string `json:"thumb_1024"`
+ Thumb1024W int `json:"thumb_1024_w"`
+ Thumb1024H int `json:"thumb_1024_h"`
+ OriginalW int `json:"original_w"`
+ OriginalH int `json:"original_h"`
+ ThumbTiny string `json:"thumb_tiny"`
+ Permalink string `json:"permalink"`
+ PermalinkPublic string `json:"permalink_public"`
+ IsStarred bool `json:"is_starred"`
+ HasRichPreview bool `json:"has_rich_preview"`
+ FileAccess string `json:"file_access"`
+ } `json:"files"`
+ Upload bool `json:"upload"`
+ Blocks []struct {
+ Type string `json:"type"`
+ BlockId string `json:"block_id"`
+ Elements []struct {
+ Type string `json:"type"`
+ Elements []struct {
+ Type string `json:"type"`
+ Text string `json:"text"`
+ Style struct {
+ Bold bool `json:"bold"`
+ } `json:"style,omitempty"`
+ } `json:"elements"`
+ } `json:"elements"`
+ } `json:"blocks"`
+
+ Root struct {
+ ClientMsgId string `json:"client_msg_id"`
+ Type string `json:"type"`
+ Text string `json:"text"`
+ User string `json:"user"`
+ Ts string `json:"ts"`
+ Blocks []struct {
+ Type string `json:"type"`
+ BlockId string `json:"block_id"`
+ Elements []struct {
+ Type string `json:"type"`
+ Elements []struct {
+ Type string `json:"type"`
+ Text string `json:"text,omitempty"`
+ Name string `json:"name,omitempty"`
+ Unicode string
`json:"unicode,omitempty"`
+ } `json:"elements"`
+ } `json:"elements"`
+ } `json:"blocks"`
+ Team string `json:"team"`
+ ThreadTs string `json:"thread_ts"`
+ ReplyCount int `json:"reply_count"`
+ ReplyUsersCount int `json:"reply_users_count"`
+ LatestReply string `json:"latest_reply"`
+ ReplyUsers []string `json:"reply_users"`
+ IsLocked bool `json:"is_locked"`
+ Subscribed bool `json:"subscribed"`
+ } `json:"root"`
+ Reactions []struct {
+ Name string `json:"name"`
+ Users []string `json:"users"`
+ Count int `json:"count"`
+ } `json:"reactions"`
+}
+
+type SlackThreadsApiResult struct {
+ Ok bool `json:"ok"`
+ Threads []json.RawMessage `json:"messages"`
+ ResponseMetadata struct {
+ NextCursor string `json:"next_cursor"`
+ } `json:"response_metadata"`
+}
diff --git a/backend/plugins/slack/impl/impl.go
b/backend/plugins/slack/impl/impl.go
new file mode 100644
index 000000000..896f7955f
--- /dev/null
+++ b/backend/plugins/slack/impl/impl.go
@@ -0,0 +1,130 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package impl
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/slack/api"
+ "github.com/apache/incubator-devlake/plugins/slack/models"
+
"github.com/apache/incubator-devlake/plugins/slack/models/migrationscripts"
+ "github.com/apache/incubator-devlake/plugins/slack/tasks"
+)
+
+var _ plugin.PluginMeta = (*Slack)(nil)
+var _ plugin.PluginInit = (*Slack)(nil)
+var _ plugin.PluginTask = (*Slack)(nil)
+var _ plugin.PluginApi = (*Slack)(nil)
+var _ plugin.PluginModel = (*Slack)(nil)
+var _ plugin.PluginMigration = (*Slack)(nil)
+var _ plugin.CloseablePluginTask = (*Slack)(nil)
+
+type Slack struct{}
+
+func (p Slack) Init(basicRes context.BasicRes) errors.Error {
+ api.Init(basicRes)
+ return nil
+}
+
+func (p Slack) GetTablesInfo() []dal.Tabler {
+ return []dal.Tabler{
+ &models.SlackConnection{},
+ }
+}
+
+func (p Slack) Description() string {
+ return "To collect and enrich data from Slack"
+}
+
+func (p Slack) SubTaskMetas() []plugin.SubTaskMeta {
+ return []plugin.SubTaskMeta{
+ tasks.CollectChannelMeta,
+ tasks.ExtractChannelMeta,
+
+ tasks.CollectChannelMessageMeta,
+ tasks.ExtractChannelMessageMeta,
+
+ tasks.CollectThreadMeta,
+ tasks.ExtractThreadMeta,
+ }
+}
+
+func (p Slack) PrepareTaskData(taskCtx plugin.TaskContext, options
map[string]interface{}) (interface{}, errors.Error) {
+ var op tasks.SlackOptions
+ if err := helper.Decode(options, &op, nil); err != nil {
+ return nil, err
+ }
+
+ connectionHelper := helper.NewConnectionHelper(
+ taskCtx,
+ nil,
+ )
+ connection := &models.SlackConnection{}
+ err := connectionHelper.FirstById(connection, op.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+
+ apiClient, err := tasks.NewSlackApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, err
+ }
+ return &tasks.SlackTaskData{
+ Options: &op,
+ ApiClient: apiClient,
+ }, nil
+}
+
+func (p Slack) RootPkgPath() string {
+ return "github.com/apache/incubator-devlake/plugins/slack"
+}
+
+func (p Slack) MigrationScripts() []plugin.MigrationScript {
+ return migrationscripts.All()
+}
+
+func (p Slack) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
+ return map[string]map[string]plugin.ApiResourceHandler{
+ "test": {
+ "POST": api.TestConnection,
+ },
+ "connections": {
+ "POST": api.PostConnections,
+ "GET": api.ListConnections,
+ },
+ "connections/:connectionId": {
+ "PATCH": api.PatchConnection,
+ "DELETE": api.DeleteConnection,
+ "GET": api.GetConnection,
+ },
+ }
+}
+
+func (p Slack) Close(taskCtx plugin.TaskContext) errors.Error {
+ data, ok := taskCtx.GetData().(*tasks.SlackTaskData)
+ if !ok {
+ return errors.Default.New(fmt.Sprintf("GetData failed when try
to close %+v", taskCtx))
+ }
+ data.ApiClient.Release()
+ return nil
+}
diff --git a/backend/plugins/slack/models/channel.go
b/backend/plugins/slack/models/channel.go
new file mode 100644
index 000000000..ba8aa1724
--- /dev/null
+++ b/backend/plugins/slack/models/channel.go
@@ -0,0 +1,52 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/core/models/common"
+)
+
+type SlackChannel struct {
+ common.NoPKModel `json:"-"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Id string `json:"id" gorm:"primaryKey"`
+ Name string `json:"name"`
+ IsChannel bool `json:"is_channel"`
+ IsGroup bool `json:"is_group"`
+ IsIm bool `json:"is_im"`
+ IsMpim bool `json:"is_mpim"`
+ IsPrivate bool `json:"is_private"`
+ Created int `json:"created"`
+ IsArchived bool `json:"is_archived"`
+ IsGeneral bool `json:"is_general"`
+ Unlinked int `json:"unlinked"`
+ NameNormalized string `json:"name_normalized"`
+ IsShared bool `json:"is_shared"`
+ IsOrgShared bool `json:"is_org_shared"`
+ IsPendingExtShared bool `json:"is_pending_ext_shared"`
+ ContextTeamId string `json:"context_team_id"`
+ Updated int64 `json:"updated"`
+ Creator string `json:"creator"`
+ IsExtShared bool `json:"is_ext_shared"`
+ IsMember bool `json:"is_member"`
+ NumMembers int `json:"num_members"`
+}
+
+func (SlackChannel) TableName() string {
+ return "_tool_slack_channels"
+}
diff --git a/backend/plugins/slack/models/channel_message.go
b/backend/plugins/slack/models/channel_message.go
new file mode 100644
index 000000000..7e93b56fa
--- /dev/null
+++ b/backend/plugins/slack/models/channel_message.go
@@ -0,0 +1,46 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/core/models/common"
+)
+
+type SlackChannelMessage struct {
+ common.NoPKModel `json:"-"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ChannelId string `json:"channel_id" gorm:"primaryKey"`
+ Ts string `json:"ts" gorm:"primaryKey"`
+ ClientMsgId string `json:"client_msg_id"`
+ Type string `json:"type"`
+ Subtype string `json:"subtype"`
+ ThreadTs string `json:"thread_ts"`
+ User string `json:"user"`
+ Text string `json:"text"`
+ Team string `json:"team"`
+ ReplyCount int `json:"reply_count"`
+ ReplyUsersCount int `json:"reply_users_count"`
+ LatestReply string `json:"latest_reply"`
+ IsLocked bool `json:"is_locked"`
+ Subscribed bool `json:"subscribed"`
+ ParentUserId string `json:"parent_user_id"`
+}
+
+func (SlackChannelMessage) TableName() string {
+ return "_tool_slack_channel_messages"
+}
diff --git a/backend/plugins/slack/models/connection.go
b/backend/plugins/slack/models/connection.go
new file mode 100644
index 000000000..89182e35b
--- /dev/null
+++ b/backend/plugins/slack/models/connection.go
@@ -0,0 +1,38 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+// SlackConn holds the essential information to connect to the Slack API
+type SlackConn struct {
+ helper.RestConnection `mapstructure:",squash"`
+ helper.AccessToken `mapstructure:",squash"`
+}
+
+// SlackConnection holds SlackConn plus ID/Name for database storage
+type SlackConnection struct {
+ helper.BaseConnection `mapstructure:",squash"`
+ SlackConn `mapstructure:",squash"`
+}
+
+func (SlackConnection) TableName() string {
+ return "_tool_slack_connections"
+}
diff --git
a/backend/plugins/slack/models/migrationscripts/20230421_add_init_tables.go
b/backend/plugins/slack/models/migrationscripts/20230421_add_init_tables.go
new file mode 100644
index 000000000..21e322914
--- /dev/null
+++ b/backend/plugins/slack/models/migrationscripts/20230421_add_init_tables.go
@@ -0,0 +1,78 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+
"github.com/apache/incubator-devlake/plugins/slack/models/migrationscripts/archived"
+)
+
+type addInitTables struct {
+}
+
+func (u *addInitTables) Up(basicRes context.BasicRes) errors.Error {
+ db := basicRes.GetDal()
+
+ err := db.DropTables(
+ &archived.SlackConnection{},
+ &archived.SlackChannel{},
+ &archived.SlackChannelMessage{},
+ )
+ if err != nil {
+ return err
+ }
+
+ err = migrationhelper.AutoMigrateTables(
+ basicRes,
+ &archived.SlackConnection{},
+ &archived.SlackChannel{},
+ &archived.SlackChannelMessage{},
+ )
+ if err != nil {
+ return err
+ }
+
+ encodeKey := basicRes.GetConfig(plugin.EncodeKeyEnvStr)
+ connection := &archived.SlackConnection{}
+ connection.Endpoint = basicRes.GetConfig(`SLACK_ENDPOINT`)
+ connection.Token = basicRes.GetConfig(`SLACK_TOKEN`)
+ connection.Name = `Slack`
+ if connection.Endpoint != `` && connection.Token != `` && encodeKey !=
`` {
+ connection.Token, err = plugin.Encrypt(encodeKey,
connection.Token)
+ if err != nil {
+ return err
+ }
+ // update from .env and save to db
+ err = db.CreateIfNotExist(connection)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (*addInitTables) Version() uint64 {
+ return 20230421000001
+}
+
+func (*addInitTables) Name() string {
+ return "Slack init schemas"
+}
diff --git a/backend/plugins/slack/models/migrationscripts/archived/channel.go
b/backend/plugins/slack/models/migrationscripts/archived/channel.go
new file mode 100644
index 000000000..c7475299a
--- /dev/null
+++ b/backend/plugins/slack/models/migrationscripts/archived/channel.go
@@ -0,0 +1,52 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+)
+
+type SlackChannel struct {
+ archived.NoPKModel `json:"-"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Id string `json:"id" gorm:"primaryKey"`
+ Name string `json:"name"`
+ IsChannel bool `json:"is_channel"`
+ IsGroup bool `json:"is_group"`
+ IsIm bool `json:"is_im"`
+ IsMpim bool `json:"is_mpim"`
+ IsPrivate bool `json:"is_private"`
+ Created int `json:"created"`
+ IsArchived bool `json:"is_archived"`
+ IsGeneral bool `json:"is_general"`
+ Unlinked int `json:"unlinked"`
+ NameNormalized string `json:"name_normalized"`
+ IsShared bool `json:"is_shared"`
+ IsOrgShared bool `json:"is_org_shared"`
+ IsPendingExtShared bool `json:"is_pending_ext_shared"`
+ ContextTeamId string `json:"context_team_id"`
+ Updated int64 `json:"updated"`
+ Creator string `json:"creator"`
+ IsExtShared bool `json:"is_ext_shared"`
+ IsMember bool `json:"is_member"`
+ NumMembers int `json:"num_members"`
+}
+
+func (SlackChannel) TableName() string {
+ return "_tool_slack_channels"
+}
diff --git
a/backend/plugins/slack/models/migrationscripts/archived/channel_message.go
b/backend/plugins/slack/models/migrationscripts/archived/channel_message.go
new file mode 100644
index 000000000..a58b24bde
--- /dev/null
+++ b/backend/plugins/slack/models/migrationscripts/archived/channel_message.go
@@ -0,0 +1,46 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+)
+
+type SlackChannelMessage struct {
+ archived.NoPKModel `json:"-"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ChannelId string `json:"channel_id" gorm:"primaryKey"`
+ Ts string `json:"ts" gorm:"primaryKey"`
+ ClientMsgId string `json:"client_msg_id"`
+ Type string `json:"type"`
+ Subtype string `json:"subtype"`
+ ThreadTs string `json:"thread_ts"`
+ User string `json:"user"`
+ Text string `json:"text"`
+ Team string `json:"team"`
+ ReplyCount int `json:"reply_count"`
+ ReplyUsersCount int `json:"reply_users_count"`
+ LatestReply string `json:"latest_reply"`
+ IsLocked bool `json:"is_locked"`
+ Subscribed bool `json:"subscribed"`
+ ParentUserId string `json:"parent_user_id"`
+}
+
+func (SlackChannelMessage) TableName() string {
+ return "_tool_slack_channel_messages"
+}
diff --git
a/backend/plugins/slack/models/migrationscripts/archived/connection.go
b/backend/plugins/slack/models/migrationscripts/archived/connection.go
new file mode 100644
index 000000000..8a6806dce
--- /dev/null
+++ b/backend/plugins/slack/models/migrationscripts/archived/connection.go
@@ -0,0 +1,35 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ commonArchived
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+)
+
+type SlackConnection struct {
+ commonArchived.Model
+ Name string `gorm:"type:varchar(100);uniqueIndex"
json:"name" validate:"required"`
+ Endpoint string `gorm:"type:varchar(255)"`
+ Proxy string `json:"proxy" gorm:"type:varchar(255)"`
+ RateLimitPerHour int `comment:"api request rate limit per hour"`
+ Token string `gorm:"type:varchar(255)"`
+}
+
+func (SlackConnection) TableName() string {
+ return "_tool_slack_connections"
+}
diff --git a/backend/plugins/slack/models/migrationscripts/register.go
b/backend/plugins/slack/models/migrationscripts/register.go
new file mode 100644
index 000000000..ec054748c
--- /dev/null
+++ b/backend/plugins/slack/models/migrationscripts/register.go
@@ -0,0 +1,29 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+// All return all the migration scripts
+func All() []plugin.MigrationScript {
+ return []plugin.MigrationScript{
+ new(addInitTables),
+ }
+}
diff --git a/backend/plugins/slack/slack.go b/backend/plugins/slack/slack.go
new file mode 100644
index 000000000..01355bb29
--- /dev/null
+++ b/backend/plugins/slack/slack.go
@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "github.com/apache/incubator-devlake/core/runner"
+ "github.com/apache/incubator-devlake/plugins/slack/impl"
+ "github.com/spf13/cobra"
+)
+
+var PluginEntry impl.Slack
+
+// standalone mode for debugging
+func main() {
+ cmd := &cobra.Command{Use: "slack"}
+ connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "slack
connection id")
+ _ = cmd.MarkFlagRequired("connectionId")
+ cmd.Run = func(cmd *cobra.Command, args []string) {
+ runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
+ "connectionId": *connectionId,
+ })
+ }
+ runner.RunCmd(cmd)
+}
diff --git a/backend/plugins/slack/tasks/api_client.go
b/backend/plugins/slack/tasks/api_client.go
new file mode 100644
index 000000000..865259661
--- /dev/null
+++ b/backend/plugins/slack/tasks/api_client.go
@@ -0,0 +1,42 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/slack/models"
+)
+
+func NewSlackApiClient(taskCtx plugin.TaskContext, connection
*models.SlackConnection) (*api.ApiAsyncClient, errors.Error) {
+ apiClient, err := api.NewApiClientFromConnection(taskCtx.GetContext(),
taskCtx, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ // create async api client
+ asyncApiClient, err := api.CreateAsyncApiClient(taskCtx, apiClient,
&api.ApiRateLimitCalculator{
+ UserRateLimitPerHour: connection.RateLimitPerHour,
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ return asyncApiClient, nil
+}
diff --git a/backend/plugins/slack/tasks/channel_collector.go
b/backend/plugins/slack/tasks/channel_collector.go
new file mode 100644
index 000000000..d2a6f76d8
--- /dev/null
+++ b/backend/plugins/slack/tasks/channel_collector.go
@@ -0,0 +1,91 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/slack/apimodels"
+ "net/http"
+ "net/url"
+ "strconv"
+)
+
+const RAW_CHANNEL_TABLE = "slack_channel"
+
+var _ plugin.SubTaskEntryPoint = CollectChannel
+
+// CollectChannel collect all channels that bot is in
+func CollectChannel(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*SlackTaskData)
+ pageSize := 100
+ collector, err := api.NewApiCollector(api.ApiCollectorArgs{
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: SlackApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ },
+ Table: RAW_CHANNEL_TABLE,
+ },
+ ApiClient: data.ApiClient,
+ Incremental: false,
+ UrlTemplate: "conversations.list",
+ PageSize: pageSize,
+ GetNextPageCustomData: func(prevReqData *api.RequestData,
prevPageResponse *http.Response) (interface{}, errors.Error) {
+ res := apimodels.SlackChannelMessageApiResult{}
+ err := api.UnmarshalResponse(prevPageResponse, &res)
+ if err != nil {
+ return nil, err
+ }
+ if res.ResponseMetadata.NextCursor == "" {
+ return nil, api.ErrFinishCollect
+ }
+ return res.ResponseMetadata.NextCursor, nil
+ },
+ Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
+ query := url.Values{}
+ query.Set("limit", strconv.Itoa(pageSize))
+ if pageToken, ok := reqData.CustomData.(string); ok &&
pageToken != "" {
+ query.Set("cursor", reqData.CustomData.(string))
+ }
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
+ body := &apimodels.SlackChannelApiResult{}
+ err := api.UnmarshalResponse(res, body)
+ if err != nil {
+ return nil, err
+ }
+ return body.Channels, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
+
+var CollectChannelMeta = plugin.SubTaskMeta{
+ Name: "collectChannel",
+ EntryPoint: CollectChannel,
+ EnabledByDefault: true,
+ Description: "Collect channels from Slack api",
+}
diff --git a/backend/plugins/slack/tasks/channel_extractor.go
b/backend/plugins/slack/tasks/channel_extractor.go
new file mode 100644
index 000000000..2374ba0eb
--- /dev/null
+++ b/backend/plugins/slack/tasks/channel_extractor.go
@@ -0,0 +1,62 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/slack/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractChannel
+
+func ExtractChannel(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*SlackTaskData)
+ extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: SlackApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ },
+ Table: RAW_CHANNEL_TABLE,
+ },
+ Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+ body := &models.SlackChannel{}
+ err := errors.Convert(json.Unmarshal(row.Data, body))
+ if err != nil {
+ return nil, err
+ }
+ body.ConnectionId = data.Options.ConnectionId
+ return []interface{}{body}, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+var ExtractChannelMeta = plugin.SubTaskMeta{
+ Name: "extractChannel",
+ EntryPoint: ExtractChannel,
+ EnabledByDefault: true,
+ Description: "Extract raw channel data into tool layer table",
+}
diff --git a/backend/plugins/slack/tasks/channel_message_collector.go
b/backend/plugins/slack/tasks/channel_message_collector.go
new file mode 100644
index 000000000..7c12391d6
--- /dev/null
+++ b/backend/plugins/slack/tasks/channel_message_collector.go
@@ -0,0 +1,118 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/slack/apimodels"
+ "net/http"
+ "net/url"
+ "reflect"
+ "strconv"
+)
+
+const RAW_CHANNEL_MESSAGE_TABLE = "slack_channel_message"
+
+var _ plugin.SubTaskEntryPoint = CollectChannelMessage
+
+type ChannelInput struct {
+ ChannelId string `json:"channel_id"`
+}
+
+func CollectChannelMessage(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*SlackTaskData)
+ db := taskCtx.GetDal()
+
+ clauses := []dal.Clause{
+ dal.Select("id as channel_id"),
+ dal.From("_tool_slack_channels"),
+ dal.Where("connection_id=?", data.Options.ConnectionId),
+ }
+
+ // construct the input iterator
+ cursor, err := db.Cursor(clauses...)
+ if err != nil {
+ return err
+ }
+ // smaller struct can reduce memory footprint, we should try to avoid
using big struct
+ iterator, err := api.NewDalCursorIterator(db, cursor,
reflect.TypeOf(ChannelInput{}))
+ if err != nil {
+ return err
+ }
+
+ pageSize := 100
+ collector, err := api.NewApiCollector(api.ApiCollectorArgs{
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: SlackApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ },
+ Table: RAW_CHANNEL_MESSAGE_TABLE,
+ },
+ ApiClient: data.ApiClient,
+ Incremental: false,
+ Input: iterator,
+ UrlTemplate: "conversations.history",
+ PageSize: pageSize,
+ GetNextPageCustomData: func(prevReqData *api.RequestData,
prevPageResponse *http.Response) (interface{}, errors.Error) {
+ res := apimodels.SlackChannelMessageApiResult{}
+ err := api.UnmarshalResponse(prevPageResponse, &res)
+ if err != nil {
+ return nil, err
+ }
+ if res.ResponseMetadata.NextCursor == "" {
+ return nil, api.ErrFinishCollect
+ }
+ return res.ResponseMetadata.NextCursor, nil
+ },
+ Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
+ input := reqData.Input.(*ChannelInput)
+ query := url.Values{}
+ query.Set("channel", input.ChannelId)
+ query.Set("limit", strconv.Itoa(pageSize))
+ if pageToken, ok := reqData.CustomData.(string); ok &&
pageToken != "" {
+ query.Set("cursor", reqData.CustomData.(string))
+ }
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
+ body := &apimodels.SlackChannelMessageApiResult{}
+ err := api.UnmarshalResponse(res, body)
+ if err != nil {
+ return nil, err
+ }
+ return body.Messages, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
+
+var CollectChannelMessageMeta = plugin.SubTaskMeta{
+ Name: "collectChannelMessage",
+ EntryPoint: CollectChannelMessage,
+ EnabledByDefault: true,
+ Description: "Collect channel message from Slack api",
+}
diff --git a/backend/plugins/slack/tasks/channel_message_extractor.go
b/backend/plugins/slack/tasks/channel_message_extractor.go
new file mode 100644
index 000000000..91a048133
--- /dev/null
+++ b/backend/plugins/slack/tasks/channel_message_extractor.go
@@ -0,0 +1,85 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/slack/apimodels"
+ "github.com/apache/incubator-devlake/plugins/slack/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractChannelMessage
+
+func ExtractChannelMessage(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*SlackTaskData)
+ extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: SlackApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ },
+ Table: RAW_CHANNEL_MESSAGE_TABLE,
+ },
+ Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+ channel := &ChannelInput{}
+ err := errors.Convert(json.Unmarshal(row.Input,
channel))
+ if err != nil {
+ return nil, err
+ }
+
+ body := &apimodels.SlackChannelMessageResultItem{}
+ err = errors.Convert(json.Unmarshal(row.Data, body))
+ if err != nil {
+ return nil, err
+ }
+ message := &models.SlackChannelMessage{}
+ message.ConnectionId = data.Options.ConnectionId
+ message.ChannelId = channel.ChannelId
+ message.ClientMsgId = body.ClientMsgId
+ message.Type = body.Type
+ message.Subtype = body.Subtype
+ message.Ts = body.Ts
+ message.ThreadTs = body.ThreadTs
+ message.User = body.User
+ message.Text = body.Text
+ message.Team = body.Team
+ message.ReplyCount = body.ReplyCount
+ message.ReplyUsersCount = body.ReplyUsersCount
+ message.LatestReply = body.LatestReply
+ message.IsLocked = body.IsLocked
+ message.Subscribed = body.Subscribed
+ message.ParentUserId = body.ParentUserId
+ return []interface{}{message}, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+var ExtractChannelMessageMeta = plugin.SubTaskMeta{
+ Name: "extractChannelMessage",
+ EntryPoint: ExtractChannelMessage,
+ EnabledByDefault: true,
+ Description: "Extract raw channel messages data into tool layer
table",
+}
diff --git a/backend/plugins/slack/tasks/task_data.go
b/backend/plugins/slack/tasks/task_data.go
new file mode 100644
index 000000000..41c811fb6
--- /dev/null
+++ b/backend/plugins/slack/tasks/task_data.go
@@ -0,0 +1,35 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+type SlackApiParams struct {
+ ConnectionId uint64 `json:"connectionId"`
+}
+
+type SlackOptions struct {
+ ConnectionId uint64 `json:"connectionId"`
+}
+
+type SlackTaskData struct {
+ Options *SlackOptions
+ ApiClient *helper.ApiAsyncClient
+}
diff --git a/backend/plugins/slack/tasks/thread_collector.go
b/backend/plugins/slack/tasks/thread_collector.go
new file mode 100644
index 000000000..2019636a3
--- /dev/null
+++ b/backend/plugins/slack/tasks/thread_collector.go
@@ -0,0 +1,121 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/slack/apimodels"
+ "net/http"
+ "net/url"
+ "reflect"
+ "strconv"
+)
+
+const RAW_THREAD_TABLE = "slack_thread"
+
+var _ plugin.SubTaskEntryPoint = CollectThread
+
+type ThreadInput struct {
+ ChannelId string `json:"channel_id"`
+ ThreadTs string `json:"thread_ts"`
+}
+
+func CollectThread(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*SlackTaskData)
+ db := taskCtx.GetDal()
+
+ clauses := []dal.Clause{
+ dal.Select("thread_ts, channel_id"),
+ dal.From("_tool_slack_channel_messages"),
+ dal.Where("connection_id=? AND thread_ts!='' AND subtype=''",
data.Options.ConnectionId),
+ }
+
+ // construct the input iterator
+ cursor, err := db.Cursor(clauses...)
+ if err != nil {
+ return err
+ }
+ // smaller struct can reduce memory footprint, we should try to avoid
using big struct
+ iterator, err := api.NewDalCursorIterator(db, cursor,
reflect.TypeOf(ThreadInput{}))
+ if err != nil {
+ return err
+ }
+
+ pageSize := 50
+ collector, err := api.NewApiCollector(api.ApiCollectorArgs{
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: SlackApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ },
+ Table: RAW_THREAD_TABLE,
+ },
+ ApiClient: data.ApiClient,
+ Incremental: false,
+ Input: iterator,
+ UrlTemplate: "conversations.replies",
+ PageSize: pageSize,
+ GetNextPageCustomData: func(prevReqData *api.RequestData,
prevPageResponse *http.Response) (interface{}, errors.Error) {
+ res := apimodels.SlackThreadsApiResult{}
+ err := api.UnmarshalResponse(prevPageResponse, &res)
+ if err != nil {
+ return nil, err
+ }
+ if res.ResponseMetadata.NextCursor == "" {
+ return nil, api.ErrFinishCollect
+ }
+ return res.ResponseMetadata.NextCursor, nil
+ },
+ Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
+ input := reqData.Input.(*ThreadInput)
+ query := url.Values{}
+ query.Set("channel", input.ChannelId)
+ query.Set("ts", input.ThreadTs)
+ query.Set("offset", strconv.Itoa(reqData.Pager.Skip))
+ query.Set("limit", strconv.Itoa(pageSize))
+ if pageToken, ok := reqData.CustomData.(string); ok &&
pageToken != "" {
+ query.Set("cursor", reqData.CustomData.(string))
+ }
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
+ body := &apimodels.SlackThreadsApiResult{}
+ err := api.UnmarshalResponse(res, body)
+ if err != nil {
+ return nil, err
+ }
+ return body.Threads, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
+
+var CollectThreadMeta = plugin.SubTaskMeta{
+ Name: "collectThread",
+ EntryPoint: CollectThread,
+ EnabledByDefault: true,
+ Description: "Collect thread from Slack api",
+}
diff --git a/backend/plugins/slack/tasks/thread_extractor.go
b/backend/plugins/slack/tasks/thread_extractor.go
new file mode 100644
index 000000000..c9a9c08f0
--- /dev/null
+++ b/backend/plugins/slack/tasks/thread_extractor.go
@@ -0,0 +1,85 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/slack/apimodels"
+ "github.com/apache/incubator-devlake/plugins/slack/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractThread
+
+func ExtractThread(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*SlackTaskData)
+ extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: SlackApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ },
+ Table: RAW_THREAD_TABLE,
+ },
+ Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+ threadInput := &ThreadInput{}
+ err := errors.Convert(json.Unmarshal(row.Input,
threadInput))
+ if err != nil {
+ return nil, err
+ }
+
+ body := &apimodels.SlackChannelMessageResultItem{}
+ err = errors.Convert(json.Unmarshal(row.Data, body))
+ if err != nil {
+ return nil, err
+ }
+ message := &models.SlackChannelMessage{}
+ message.ConnectionId = data.Options.ConnectionId
+ message.ChannelId = threadInput.ChannelId
+ message.ClientMsgId = body.ClientMsgId
+ message.Type = body.Type
+ message.Subtype = body.Subtype
+ message.Ts = body.Ts
+ message.ThreadTs = body.ThreadTs
+ message.User = body.User
+ message.Text = body.Text
+ message.Team = body.Team
+ message.ReplyCount = body.ReplyCount
+ message.ReplyUsersCount = body.ReplyUsersCount
+ message.LatestReply = body.LatestReply
+ message.IsLocked = body.IsLocked
+ message.Subscribed = body.Subscribed
+ message.ParentUserId = body.ParentUserId
+ return []interface{}{message}, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+var ExtractThreadMeta = plugin.SubTaskMeta{
+ Name: "extractThread",
+ EntryPoint: ExtractThread,
+ EnabledByDefault: true,
+ Description: "Extract raw thread messages data into tool layer
table",
+}
