This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new f1543613a feat:support comlexity data scope (#8593)
f1543613a is described below
commit f1543613a9673acb64d7b4da549a10131ae275a6
Author: Warren Chen <[email protected]>
AuthorDate: Fri Sep 26 19:58:09 2025 +0800
feat:support comlexity data scope (#8593)
* feat:support comlexicy data scope
* fix: scopedata
* fix: unit test
---
backend/plugins/q_dev/api/init.go | 14 +
backend/plugins/q_dev/api/s3_slice_api.go | 121 +++++++
backend/plugins/q_dev/impl/impl.go | 15 +-
backend/plugins/q_dev/impl/impl_test.go | 2 +-
.../20250926_add_s3_slice_table.go} | 29 +-
.../{register.go => archived/s3_slice.go} | 24 +-
.../q_dev/models/migrationscripts/register.go | 1 +
backend/plugins/q_dev/models/s3_slice.go | 231 ++++++++++++
config-ui/src/plugins/register/q-dev/config.tsx | 9 +
.../src/plugins/register/q-dev/data-scope.tsx | 400 +++++++++++++++++++++
10 files changed, 821 insertions(+), 25 deletions(-)
diff --git a/backend/plugins/q_dev/api/init.go
b/backend/plugins/q_dev/api/init.go
index 7a16d5f0c..3bb67450b 100644
--- a/backend/plugins/q_dev/api/init.go
+++ b/backend/plugins/q_dev/api/init.go
@@ -21,12 +21,15 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/srvhelper"
+ "github.com/apache/incubator-devlake/plugins/q_dev/models"
"github.com/go-playground/validator/v10"
)
var vld *validator.Validate
var connectionHelper *api.ConnectionApiHelper
var basicRes context.BasicRes
+var dsHelper *api.DsHelper[models.QDevConnection, models.QDevS3Slice,
srvhelper.NoScopeConfig]
func Init(br context.BasicRes, p plugin.PluginMeta) {
basicRes = br
@@ -36,4 +39,15 @@ func Init(br context.BasicRes, p plugin.PluginMeta) {
vld,
p.Name(),
)
+
+ dsHelper = api.NewDataSourceHelper[
+ models.QDevConnection, models.QDevS3Slice,
srvhelper.NoScopeConfig,
+ ](
+ basicRes,
+ p.Name(),
+ []string{"prefix", "basePath", "name"},
+ func(c models.QDevConnection) models.QDevConnection { return
c.Sanitize() },
+ func(s models.QDevS3Slice) models.QDevS3Slice { return
s.Sanitize() },
+ nil,
+ )
}
diff --git a/backend/plugins/q_dev/api/s3_slice_api.go
b/backend/plugins/q_dev/api/s3_slice_api.go
new file mode 100644
index 000000000..737081302
--- /dev/null
+++ b/backend/plugins/q_dev/api/s3_slice_api.go
@@ -0,0 +1,121 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/srvhelper"
+ "github.com/apache/incubator-devlake/plugins/q_dev/models"
+)
+
+type PutScopesReqBody = helper.PutScopesReqBody[models.QDevS3Slice]
+type ScopeDetail = srvhelper.ScopeDetail[models.QDevS3Slice,
srvhelper.NoScopeConfig]
+
+// PutScopes create or update Q Developer scopes (S3 prefixes)
+// @Summary create or update Q Developer scopes
+// @Description Create or update Q Developer scopes
+// @Tags plugins/q_dev
+// @Accept application/json
+// @Param connectionId path int true "connection ID"
+// @Param scope body PutScopesReqBody true "json"
+// @Success 200 {object} []models.QDevS3Slice
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/q_dev/connections/{connectionId}/scopes [PUT]
+func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return dsHelper.ScopeApi.PutMultiple(input)
+}
+
+// GetScopeList returns Q Developer scopes
+// @Summary get Q Developer scopes
+// @Description get Q Developer scopes
+// @Tags plugins/q_dev
+// @Param connectionId path int true "connection ID"
+// @Param pageSize query int false "page size"
+// @Param page query int false "page number"
+// @Param blueprints query bool false "include blueprint references"
+// @Success 200 {object} []ScopeDetail
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/q_dev/connections/{connectionId}/scopes [GET]
+func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return dsHelper.ScopeApi.GetPage(input)
+}
+
+// GetScope returns a single scope record
+// @Summary get a Q Developer scope
+// @Description get a Q Developer scope
+// @Tags plugins/q_dev
+// @Param connectionId path int true "connection ID"
+// @Param scopeId path string true "scope id"
+// @Param blueprints query bool false "include blueprint references"
+// @Success 200 {object} ScopeDetail
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/q_dev/connections/{connectionId}/scopes/{scopeId} [GET]
+func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return dsHelper.ScopeApi.GetScopeDetail(input)
+}
+
+// PatchScope updates a scope record
+// @Summary patch a Q Developer scope
+// @Description patch a Q Developer scope
+// @Tags plugins/q_dev
+// @Accept application/json
+// @Param connectionId path int true "connection ID"
+// @Param scopeId path string true "scope id"
+// @Param scope body models.QDevS3Slice true "json"
+// @Success 200 {object} models.QDevS3Slice
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/q_dev/connections/{connectionId}/scopes/{scopeId} [PATCH]
+func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return dsHelper.ScopeApi.Patch(input)
+}
+
+// DeleteScope removes a scope and optionally associated data.
+// @Summary delete a Q Developer scope
+// @Description delete Q Developer scope data
+// @Tags plugins/q_dev
+// @Param connectionId path int true "connection ID"
+// @Param scopeId path string true "scope id"
+// @Param delete_data_only query bool false "Only delete scope data"
+// @Success 200
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 409 {object} srvhelper.DsRefs "References exist to this scope"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/q_dev/connections/{connectionId}/scopes/{scopeId} [DELETE]
+func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return dsHelper.ScopeApi.Delete(input)
+}
+
+// GetScopeLatestSyncState returns scope sync state info
+// @Summary latest sync state for a Q Developer scope
+// @Description get latest sync state for a Q Developer scope
+// @Tags plugins/q_dev
+// @Param connectionId path int true "connection ID"
+// @Param scopeId path string true "scope id"
+// @Success 200
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router
/plugins/q_dev/connections/{connectionId}/scopes/{scopeId}/latest-sync-state
[GET]
+func GetScopeLatestSyncState(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ return dsHelper.ScopeApi.GetScopeLatestSyncState(input)
+}
diff --git a/backend/plugins/q_dev/impl/impl.go
b/backend/plugins/q_dev/impl/impl.go
index b2568334b..9c3824924 100644
--- a/backend/plugins/q_dev/impl/impl.go
+++ b/backend/plugins/q_dev/impl/impl.go
@@ -54,6 +54,7 @@ func (p QDev) GetTablesInfo() []dal.Tabler {
&models.QDevConnection{},
&models.QDevUserData{},
&models.QDevS3FileMeta{},
+ &models.QDevS3Slice{},
}
}
@@ -70,7 +71,7 @@ func (p QDev) Connection() dal.Tabler {
}
func (p QDev) Scope() plugin.ToolLayerScope {
- return nil
+ return &models.QDevS3Slice{}
}
func (p QDev) ScopeConfig() dal.Tabler {
@@ -146,6 +147,18 @@ func (p QDev) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
"connections/:connectionId/test": {
"POST": api.TestExistingConnection,
},
+ "connections/:connectionId/scopes": {
+ "GET": api.GetScopeList,
+ "PUT": api.PutScopes,
+ },
+ "connections/:connectionId/scopes/:scopeId": {
+ "GET": api.GetScope,
+ "PATCH": api.PatchScope,
+ "DELETE": api.DeleteScope,
+ },
+ "connections/:connectionId/scopes/:scopeId/latest-sync-state": {
+ "GET": api.GetScopeLatestSyncState,
+ },
}
}
diff --git a/backend/plugins/q_dev/impl/impl_test.go
b/backend/plugins/q_dev/impl/impl_test.go
index 7c02ef488..97dea86e6 100644
--- a/backend/plugins/q_dev/impl/impl_test.go
+++ b/backend/plugins/q_dev/impl/impl_test.go
@@ -34,7 +34,7 @@ func TestQDev_BasicPluginMethods(t *testing.T) {
// Test table info
tables := plugin.GetTablesInfo()
- assert.Len(t, tables, 3)
+ assert.Len(t, tables, 4)
// Test subtask metas
subtasks := plugin.SubTaskMetas()
diff --git a/backend/plugins/q_dev/api/init.go
b/backend/plugins/q_dev/models/migrationscripts/20250926_add_s3_slice_table.go
similarity index 60%
copy from backend/plugins/q_dev/api/init.go
copy to
backend/plugins/q_dev/models/migrationscripts/20250926_add_s3_slice_table.go
index 7a16d5f0c..19fe467b1 100644
--- a/backend/plugins/q_dev/api/init.go
+++
b/backend/plugins/q_dev/models/migrationscripts/20250926_add_s3_slice_table.go
@@ -15,25 +15,28 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package api
+package migrationscripts
import (
"github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/go-playground/validator/v10"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+
"github.com/apache/incubator-devlake/plugins/q_dev/models/migrationscripts/archived"
)
-var vld *validator.Validate
-var connectionHelper *api.ConnectionApiHelper
-var basicRes context.BasicRes
+type addS3SliceTable struct{}
-func Init(br context.BasicRes, p plugin.PluginMeta) {
- basicRes = br
- vld = validator.New()
- connectionHelper = api.NewConnectionHelper(
+func (*addS3SliceTable) Up(basicRes context.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(
basicRes,
- vld,
- p.Name(),
+ &archived.QDevS3Slice{},
)
}
+
+func (*addS3SliceTable) Version() uint64 {
+ return 20250926
+}
+
+func (*addS3SliceTable) Name() string {
+ return "Add S3 slice table for QDev plugin"
+}
diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go
b/backend/plugins/q_dev/models/migrationscripts/archived/s3_slice.go
similarity index 61%
copy from backend/plugins/q_dev/models/migrationscripts/register.go
copy to backend/plugins/q_dev/models/migrationscripts/archived/s3_slice.go
index 85a74690e..77c7dec0d 100644
--- a/backend/plugins/q_dev/models/migrationscripts/register.go
+++ b/backend/plugins/q_dev/models/migrationscripts/archived/s3_slice.go
@@ -15,18 +15,22 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package archived
import (
- "github.com/apache/incubator-devlake/core/plugin"
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
)
-// All return all migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(initTables),
- new(modifyFileMetaTable),
- new(addDisplayNameFields),
- new(addMissingMetrics),
- }
+type QDevS3Slice struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Id string `gorm:"primaryKey;type:varchar(512)"`
+ Prefix string `gorm:"type:varchar(512);not null"`
+ BasePath string `gorm:"type:varchar(512)"`
+ Year int `gorm:"not null"`
+ Month *int
+}
+
+func (QDevS3Slice) TableName() string {
+ return "_tool_q_dev_s3_slices"
}
diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go
b/backend/plugins/q_dev/models/migrationscripts/register.go
index 85a74690e..dfff79fae 100644
--- a/backend/plugins/q_dev/models/migrationscripts/register.go
+++ b/backend/plugins/q_dev/models/migrationscripts/register.go
@@ -28,5 +28,6 @@ func All() []plugin.MigrationScript {
new(modifyFileMetaTable),
new(addDisplayNameFields),
new(addMissingMetrics),
+ new(addS3SliceTable),
}
}
diff --git a/backend/plugins/q_dev/models/s3_slice.go
b/backend/plugins/q_dev/models/s3_slice.go
new file mode 100644
index 000000000..c844d1634
--- /dev/null
+++ b/backend/plugins/q_dev/models/s3_slice.go
@@ -0,0 +1,231 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "fmt"
+ "strconv"
+ "strings"
+
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "gorm.io/gorm"
+)
+
+// QDevS3Slice describes a time-sliced S3 prefix to collect from.
+type QDevS3Slice struct {
+ common.Scope `mapstructure:",squash"`
+ Id string `json:"id" mapstructure:"id"
gorm:"primaryKey;type:varchar(512)"`
+ Prefix string `json:"prefix" mapstructure:"prefix"
gorm:"type:varchar(512);not null"`
+ BasePath string `json:"basePath" mapstructure:"basePath"
gorm:"type:varchar(512)"`
+ Year int `json:"year" mapstructure:"year" gorm:"not null"`
+ Month *int `json:"month,omitempty" mapstructure:"month"`
+
+ Name string `json:"name" mapstructure:"name" gorm:"-"`
+ FullName string `json:"fullName" mapstructure:"fullName" gorm:"-"`
+}
+
+func (QDevS3Slice) TableName() string {
+ return "_tool_q_dev_s3_slices"
+}
+
+// BeforeSave ensures derived fields stay in sync before persisting.
+func (s *QDevS3Slice) BeforeSave(_ *gorm.DB) error {
+ return s.normalize(true)
+}
+
+// AfterFind fills derived fields for API responses.
+func (s *QDevS3Slice) AfterFind(_ *gorm.DB) error {
+ return s.normalize(false)
+}
+
+// normalize trims inputs, derives prefix/id/name fields, and optionally
validates.
+func (s *QDevS3Slice) normalize(strict bool) error {
+ if s == nil {
+ return nil
+ }
+
+ s.BasePath = cleanPath(s.BasePath)
+ s.Prefix = cleanPath(selectNonEmpty(s.Prefix, s.Id))
+
+ if s.Year <= 0 {
+ if err := s.deriveYearAndMonthFromPrefix(); err != nil &&
strict {
+ return err
+ }
+ }
+
+ if s.Year <= 0 {
+ if strict {
+ return fmt.Errorf("year is required for QDev S3 slice")
+ }
+ }
+
+ if s.Month != nil {
+ if *s.Month < 1 || *s.Month > 12 {
+ return fmt.Errorf("month must be between 1 and 12")
+ }
+ }
+
+ if s.Prefix == "" {
+ s.Prefix = buildPrefix(s.BasePath, s.Year, s.Month)
+ }
+
+ prefix := buildPrefix(s.BasePath, s.Year, s.Month)
+ if prefix != "" {
+ s.Prefix = prefix
+ }
+
+ if s.Id == "" {
+ s.Id = s.Prefix
+ }
+
+ if s.Month != nil {
+ s.Name = fmt.Sprintf("%04d-%02d", s.Year, *s.Month)
+ } else if s.Year > 0 {
+ s.Name = fmt.Sprintf("%04d", s.Year)
+ }
+
+ if s.FullName == "" {
+ s.FullName = s.Prefix
+ }
+
+ return nil
+}
+
+func (s *QDevS3Slice) deriveYearAndMonthFromPrefix() error {
+ if s == nil {
+ return nil
+ }
+ segments := splitPath(s.Prefix)
+ if len(segments) == 0 {
+ return fmt.Errorf("prefix is empty")
+ }
+ last := segments[len(segments)-1]
+ if len(last) == 2 {
+ if month, err := strconv.Atoi(last); err == nil {
+ s.Month = ptr(month)
+ if len(segments) >= 2 {
+ yearSegment := segments[len(segments)-2]
+ year, yearErr := strconv.Atoi(yearSegment)
+ if yearErr != nil {
+ return yearErr
+ }
+ s.Year = year
+ base := segments[:len(segments)-2]
+ s.BasePath = strings.Join(base, "/")
+ return nil
+ }
+ }
+ }
+ if year, err := strconv.Atoi(last); err == nil {
+ s.Year = year
+ base := segments[:len(segments)-1]
+ s.BasePath = strings.Join(base, "/")
+ s.Month = nil
+ return nil
+ }
+ return fmt.Errorf("unable to derive year/month from prefix %q",
s.Prefix)
+}
+
+func (s QDevS3Slice) ScopeId() string {
+ return s.Id
+}
+
+func (s QDevS3Slice) ScopeName() string {
+ if s.Name != "" {
+ return s.Name
+ }
+ if s.Month != nil {
+ return fmt.Sprintf("%04d-%02d", s.Year, *s.Month)
+ }
+ if s.Year > 0 {
+ return fmt.Sprintf("%04d", s.Year)
+ }
+ return s.Prefix
+}
+
+func (s QDevS3Slice) ScopeFullName() string {
+ if s.FullName != "" {
+ return s.FullName
+ }
+ return s.Prefix
+}
+
+func (s QDevS3Slice) ScopeParams() interface{} {
+ return &QDevS3SliceParams{
+ ConnectionId: s.ConnectionId,
+ Prefix: s.Prefix,
+ }
+}
+
+// Sanitize returns a copy ready for JSON serialization.
+func (s QDevS3Slice) Sanitize() QDevS3Slice {
+ _ = s.normalize(false)
+ return s
+}
+
+type QDevS3SliceParams struct {
+ ConnectionId uint64 `json:"connectionId"`
+ Prefix string `json:"prefix"`
+}
+
+var _ plugin.ToolLayerScope = (*QDevS3Slice)(nil)
+
+func buildPrefix(basePath string, year int, month *int) string {
+ parts := splitPath(basePath)
+ if year > 0 {
+ parts = append(parts, fmt.Sprintf("%04d", year))
+ }
+ if month != nil {
+ parts = append(parts, fmt.Sprintf("%02d", *month))
+ }
+ return strings.Join(parts, "/")
+}
+
+func splitPath(value string) []string {
+ if value == "" {
+ return nil
+ }
+ chunks := strings.Split(value, "/")
+ result := make([]string, 0, len(chunks))
+ for _, chunk := range chunks {
+ trimmed := strings.TrimSpace(chunk)
+ if trimmed == "" {
+ continue
+ }
+ result = append(result, trimmed)
+ }
+ return result
+}
+
+func cleanPath(value string) string {
+ return strings.Join(splitPath(value), "/")
+}
+
+func selectNonEmpty(values ...string) string {
+ for _, v := range values {
+ if strings.TrimSpace(v) != "" {
+ return strings.TrimSpace(v)
+ }
+ }
+ return ""
+}
+
+func ptr[T any](value T) *T {
+ return &value
+}
diff --git a/config-ui/src/plugins/register/q-dev/config.tsx
b/config-ui/src/plugins/register/q-dev/config.tsx
index 3c0f71781..cc1e89511 100644
--- a/config-ui/src/plugins/register/q-dev/config.tsx
+++ b/config-ui/src/plugins/register/q-dev/config.tsx
@@ -20,6 +20,7 @@ import { IPluginConfig } from '@/types';
import Icon from './assets/icon.svg?react';
import { AwsCredentials, IdentityCenterConfig, S3Config } from
'./connection-fields';
+import { QDevDataScope } from './data-scope';
export const QDevConfig: IPluginConfig = {
plugin: 'q_dev',
@@ -78,6 +79,14 @@ export const QDevConfig: IPluginConfig = {
},
dataScope: {
title: 'S3 Prefixes',
+ render: ({ connectionId, disabledItems, selectedItems,
onChangeSelectedItems }) => (
+ <QDevDataScope
+ connectionId={connectionId}
+ disabledItems={disabledItems}
+ selectedItems={selectedItems as any}
+ onChangeSelectedItems={onChangeSelectedItems}
+ />
+ ),
},
scopeConfig: {
entities: ['CROSS'],
diff --git a/config-ui/src/plugins/register/q-dev/data-scope.tsx
b/config-ui/src/plugins/register/q-dev/data-scope.tsx
new file mode 100644
index 000000000..97d641201
--- /dev/null
+++ b/config-ui/src/plugins/register/q-dev/data-scope.tsx
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { useEffect, useMemo } from 'react';
+import { Button, Checkbox, Flex, Form, Input, InputNumber, Segmented, Table,
Tooltip, Typography } from 'antd';
+import type { ColumnsType } from 'antd/es/table';
+import { DeleteOutlined } from '@ant-design/icons';
+
+interface ScopeData {
+ prefix?: string;
+ year?: number;
+ month?: number | null;
+ basePath?: string;
+}
+
+interface ScopeItem {
+ id: string;
+ name: string;
+ fullName: string;
+ data?: ScopeData;
+}
+
+interface Props {
+ connectionId: ID;
+ disabledItems?: Array<{ id: ID }>;
+ selectedItems: ScopeItem[];
+ onChangeSelectedItems: (items: ScopeItem[]) => void;
+}
+
+const CURRENT_YEAR = new Date().getUTCFullYear();
+const MONTHS = Array.from({ length: 12 }, (_, idx) => idx + 1);
+const MONTH_LABELS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug',
'Sep', 'Oct', 'Nov', 'Dec'];
+
+const DEFAULT_BASE_PATH = 'user-report/AWSLogs';
+
+const ensureLeadingZero = (value: number) => value.toString().padStart(2, '0');
+
+const normalizeBasePath = (value: string) => value.trim().replace(/^\/+/,
'').replace(/\/+$/, '');
+
+const trimTrailingSlashes = (value: string) => value.replace(/\/+$/, '');
+
+const extractScopeMeta = (item: ScopeItem) => {
+ const data = item.data ?? {};
+ const rawPrefix = data.prefix ?? item.fullName ?? item.id;
+ const prefix = typeof rawPrefix === 'string' ?
trimTrailingSlashes(rawPrefix) : '';
+ const segments = prefix ? prefix.split('/').filter(Boolean) : [];
+
+ let month = data.month ?? null;
+ if (month === undefined || month === null) {
+ const last = segments[segments.length - 1];
+ if (last && /^(0[1-9]|1[0-2])$/.test(last)) {
+ month = Number(last);
+ } else {
+ month = null;
+ }
+ }
+
+ let year = data.year;
+ if (year === undefined || year === null) {
+ const idx = month ? segments.length - 2 : segments.length - 1;
+ const candidate = idx >= 0 ? segments[idx] : undefined;
+ if (candidate && /^\d{4}$/.test(candidate)) {
+ year = Number(candidate);
+ } else {
+ year = undefined;
+ }
+ }
+
+ let baseSegments: string[];
+ if (segments.length === 0) {
+ baseSegments = [];
+ } else if (month) {
+ baseSegments = segments.slice(0, Math.max(segments.length - 2, 0));
+ } else {
+ baseSegments = segments.slice(0, Math.max(segments.length - 1, 0));
+ }
+
+ const basePath = normalizeBasePath(data.basePath ?? (baseSegments.length ?
baseSegments.join('/') : ''));
+
+ return {
+ basePath,
+ year: typeof year === 'number' ? year : null,
+ month,
+ prefix,
+ };
+};
+
+const deriveBasePathFromSelection = (items: ScopeItem[]) => {
+ for (const item of items) {
+ const meta = extractScopeMeta(item);
+ if (meta.basePath !== undefined) {
+ return meta.basePath;
+ }
+ }
+ return undefined;
+};
+
+const buildPrefix = (basePath: string, year: number, month: number | null) => {
+ const segments = [] as string[];
+ const sanitizedBase = normalizeBasePath(basePath);
+ if (sanitizedBase) {
+ segments.push(sanitizedBase);
+ }
+ segments.push(String(year));
+ if (month !== null && month !== undefined) {
+ segments.push(ensureLeadingZero(month));
+ }
+ return segments.join('/');
+};
+
+const createScopeItem = (basePath: string, year: number, month: number |
null): ScopeItem => {
+ const sanitizedBase = normalizeBasePath(basePath);
+ const prefix = buildPrefix(sanitizedBase, year, month);
+ const isFullYear = month === null;
+ const name = isFullYear
+ ? `${year} (Full Year)`
+ : `${year}-${ensureLeadingZero(month as number)} (${MONTH_LABELS[(month as
number) - 1]})`;
+
+ return {
+ id: prefix,
+ name,
+ fullName: prefix,
+ data: {
+ basePath: sanitizedBase,
+ prefix,
+ year,
+ month,
+ },
+ };
+};
+
+const formatScopeLabel = (item: ScopeItem) => {
+ const meta = extractScopeMeta(item);
+ if (!meta.year) {
+ return item.name;
+ }
+
+ if (meta.month) {
+ const monthLabel = MONTH_LABELS[meta.month - 1] ??
ensureLeadingZero(meta.month);
+ return `${meta.year}-${ensureLeadingZero(meta.month)} (${monthLabel})`;
+ }
+
+ return `${meta.year} (Full Year)`;
+};
+
+const MONTH_OPTIONS = MONTHS.map((value) => ({
+ label: `${MONTH_LABELS[value - 1]} (${ensureLeadingZero(value)})`,
+ value,
+}));
+
+type FormValues = {
+ basePath: string;
+ year: number;
+ mode: 'year' | 'months';
+ months?: number[];
+};
+
+export const QDevDataScope = ({ connectionId: _connectionId, disabledItems,
selectedItems, onChangeSelectedItems }: Props) => {
+ const [form] = Form.useForm<FormValues>();
+
+ const disabledIds = useMemo(() => new Set(disabledItems?.map((it) =>
String(it.id)) ?? []), [disabledItems]);
+
+ const derivedBasePath = useMemo(
+ () => deriveBasePathFromSelection(selectedItems) ?? DEFAULT_BASE_PATH,
+ [selectedItems],
+ );
+
+ useEffect(() => {
+ if (!form.isFieldsTouched(['basePath'])) {
+ form.setFieldsValue({ basePath: derivedBasePath });
+ }
+ }, [derivedBasePath, form]);
+
+ useEffect(() => {
+ form.setFieldsValue({ mode: 'year', year: form.getFieldValue('year') ??
CURRENT_YEAR });
+ }, [form]);
+
+ const handleAdd = async () => {
+ const { basePath, year, mode, months = [] } = await form.validateFields();
+
+ const normalizedBase = normalizeBasePath(basePath ?? '');
+ const normalizedYear = Number(year);
+ if (!normalizedYear || Number.isNaN(normalizedYear)) {
+ return;
+ }
+
+ const currentIds = new Set(selectedItems.map((item) => item.id));
+ const hasFullYear = selectedItems.some((item) => {
+ const meta = extractScopeMeta(item);
+ return (
+ meta.basePath === normalizedBase &&
+ meta.year === normalizedYear &&
+ (meta.month === null || meta.month === undefined)
+ );
+ });
+
+ const additions: ScopeItem[] = [];
+
+ if (mode === 'year') {
+ if (hasFullYear) {
+ return;
+ }
+
+ const hasMonths = selectedItems.some((item) => {
+ const meta = extractScopeMeta(item);
+ return meta.basePath === normalizedBase && meta.year ===
normalizedYear && meta.month !== null;
+ });
+
+ if (hasMonths) {
+ return;
+ }
+
+ const item = createScopeItem(normalizedBase, normalizedYear, null);
+ if (!currentIds.has(item.id) && !disabledIds.has(item.id)) {
+ additions.push(item);
+ }
+ } else {
+ if (hasFullYear) {
+ return;
+ }
+
+ const uniqueMonths = Array.from(new Set(months)).map((m) =>
Number(m)).filter((m) => !Number.isNaN(m));
+ uniqueMonths.sort((a, b) => a - b);
+
+ uniqueMonths.forEach((month) => {
+ if (month < 1 || month > 12) {
+ return;
+ }
+
+ const item = createScopeItem(normalizedBase, normalizedYear, month);
+ if (currentIds.has(item.id) || disabledIds.has(item.id)) {
+ return;
+ }
+ additions.push(item);
+ });
+ }
+
+ if (!additions.length) {
+ return;
+ }
+
+ const next = [...selectedItems, ...additions];
+ next.sort((a, b) => a.id.localeCompare(b.id));
+ onChangeSelectedItems(next);
+
+ if (mode === 'months') {
+ form.setFieldsValue({ months: [] });
+ }
+ };
+
+ const handleRemove = (id: string) => {
+ onChangeSelectedItems(selectedItems.filter((item) => item.id !== id));
+ };
+
+ const columns: ColumnsType<ScopeItem> = [
+ {
+ title: 'Time Range',
+ dataIndex: 'id',
+ key: 'name',
+ render: (_: unknown, item) => formatScopeLabel(item),
+ },
+ {
+ title: 'S3 Prefix',
+ dataIndex: 'id',
+ key: 'prefix',
+ render: (_: unknown, item) => {
+ const meta = extractScopeMeta(item);
+ return <Typography.Text code>{meta.prefix}</Typography.Text>;
+ },
+ },
+ {
+ title: 'Base Path',
+ dataIndex: 'id',
+ key: 'basePath',
+ render: (_: unknown, item) => {
+ const meta = extractScopeMeta(item);
+ return meta.basePath ?
<Typography.Text>{meta.basePath}</Typography.Text> : <Typography.Text
type="secondary">(bucket root)</Typography.Text>;
+ },
+ },
+ {
+ title: '',
+ dataIndex: 'id',
+ key: 'action',
+ width: 80,
+ align: 'center',
+ render: (id: string) => (
+ <Tooltip title={disabledIds.has(id) ? 'Scope is used by existing
blueprint' : 'Remove'}>
+ <Button
+ type="text"
+ danger
+ icon={<DeleteOutlined />}
+ disabled={disabledIds.has(id)}
+ onClick={() => handleRemove(id)}
+ />
+ </Tooltip>
+ ),
+ },
+ ];
+
+ return (
+ <Flex vertical gap="middle">
+ <Typography.Paragraph type="secondary" style={{ marginBottom: 0 }}>
+ Pick which year and month prefixes DevLake should collect from your Q
Developer S3 bucket. Leave empty to
+ collect all available data.
+ </Typography.Paragraph>
+
+ <Form
+ form={form}
+ layout="inline"
+ initialValues={{
+ basePath: derivedBasePath,
+ year: CURRENT_YEAR,
+ mode: 'year',
+ months: [],
+ }}
+ onFinish={handleAdd}
+ style={{ rowGap: 16 }}
+ >
+ <Form.Item
+ label="Base Path"
+ name="basePath"
+ style={{ flex: 1 }}
+ tooltip="Common prefix in S3 between the bucket root and the year
directory"
+ >
+ <Input placeholder="user-report/AWSLogs/.../us-east-1" />
+ </Form.Item>
+
+ <Form.Item
+ label="Year"
+ name="year"
+ rules={[{ required: true, message: 'Enter year' }]}
+ style={{ width: 160 }}
+ >
+ <InputNumber min={2000} max={2100} style={{ width: '100%' }} />
+ </Form.Item>
+
+ <Form.Item name="mode" style={{ width: 180 }}>
+ <Segmented
+ options={[
+ { label: 'Full Year', value: 'year' },
+ { label: 'Specific Months', value: 'months' },
+ ]}
+ />
+ </Form.Item>
+
+ <Form.Item noStyle shouldUpdate>
+ {({ getFieldValue }) =>
+ getFieldValue('mode') === 'months' ? (
+ <Form.Item
+ name="months"
+ rules={[{ required: true, message: 'Select at least one month'
}]}
+ style={{ minWidth: 260 }}
+ >
+ <Checkbox.Group
+ options={MONTH_OPTIONS}
+ style={{ display: 'grid', gridTemplateColumns: 'repeat(4,
minmax(60px, 1fr))', gap: 8 }}
+ />
+ </Form.Item>
+ ) : null
+ }
+ </Form.Item>
+ <Form.Item>
+ <Button type="primary" htmlType="submit">
+ Add Scope
+ </Button>
+ </Form.Item>
+ </Form>
+
+ <Table
+ size="middle"
+ rowKey="id"
+ columns={columns}
+ dataSource={selectedItems}
+ pagination={false}
+ locale={{ emptyText: 'No scope selected yet.' }}
+ />
+
+ {selectedItems.length > 0 && (
+ <Typography.Paragraph type="secondary" style={{ marginBottom: 0 }}>
+ These selections will be stored as S3 prefixes and used during data
collection.
+ </Typography.Paragraph>
+ )}
+ </Flex>
+ );
+};
