This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 920bc624f feature: add q dev blueprint (#8650)
920bc624f is described below
commit 920bc624f1c04bc01e944410d413fa113ed25554
Author: Warren Chen <[email protected]>
AuthorDate: Tue Nov 25 16:02:36 2025 +0800
feature: add q dev blueprint (#8650)
* feat(plugins): add q dev blueprint
* fix: lint
---
backend/plugins/q_dev/api/blueprint_v200.go | 95 +++++++++++
backend/plugins/q_dev/impl/impl.go | 9 +
...=> 20251123_add_scope_config_id_to_s3_slice.go} | 33 +++-
.../q_dev/models/migrationscripts/register.go | 1 +
backend/plugins/q_dev/tasks/identity_client.go | 5 +
backend/plugins/q_dev/tasks/s3_data_extractor.go | 1 +
backend/plugins/q_dev/tasks/s3_file_collector.go | 1 +
.../q-dev/connection-fields/connection-test.tsx | 185 +++++++++++++++++++++
8 files changed, 321 insertions(+), 9 deletions(-)
diff --git a/backend/plugins/q_dev/api/blueprint_v200.go
b/backend/plugins/q_dev/api/blueprint_v200.go
new file mode 100644
index 000000000..a7a015982
--- /dev/null
+++ b/backend/plugins/q_dev/api/blueprint_v200.go
@@ -0,0 +1,95 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/srvhelper"
+ "github.com/apache/incubator-devlake/plugins/q_dev/models"
+ "github.com/apache/incubator-devlake/plugins/q_dev/tasks"
+)
+
+func MakeDataSourcePipelinePlanV200(
+ subtaskMetas []plugin.SubTaskMeta,
+ connectionId uint64,
+ bpScopes []*coreModels.BlueprintScope,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+ // load connection and scope from the db
+ connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
+ if err != nil {
+ return nil, nil, err
+ }
+ scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId,
bpScopes)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails,
connection)
+ if err != nil {
+ return nil, nil, err
+ }
+ scopes, err := makeScopesV200(scopeDetails, connection)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ return plan, scopes, nil
+}
+
+func makeDataSourcePipelinePlanV200(
+ subtaskMetas []plugin.SubTaskMeta,
+ scopeDetails []*srvhelper.ScopeDetail[models.QDevS3Slice,
srvhelper.NoScopeConfig],
+ connection *models.QDevConnection,
+) (coreModels.PipelinePlan, errors.Error) {
+ plan := make(coreModels.PipelinePlan, len(scopeDetails))
+ for i, scopeDetail := range scopeDetails {
+ s3Slice := scopeDetail.Scope
+ stage := plan[i]
+ if stage == nil {
+ stage = coreModels.PipelineStage{}
+ }
+
+ // construct task options for q_dev
+ op := &tasks.QDevOptions{
+ ConnectionId: s3Slice.ConnectionId,
+ S3Prefix: s3Slice.Prefix,
+ }
+
+ // Pass empty entities array to enable all subtasks
+ task, err := helper.MakePipelinePlanTask("q_dev", subtaskMetas,
[]string{}, op)
+ if err != nil {
+ return nil, err
+ }
+ stage = append(stage, task)
+ plan[i] = stage
+ }
+ return plan, nil
+}
+
+func makeScopesV200(
+ scopeDetails []*srvhelper.ScopeDetail[models.QDevS3Slice,
srvhelper.NoScopeConfig],
+ connection *models.QDevConnection,
+) ([]plugin.Scope, errors.Error) {
+ scopes := make([]plugin.Scope, 0)
+ // For Q Developer metrics, we don't need to create domain layer scopes
+ // The data is collected and stored directly in the tool layer
+ return scopes, nil
+}
diff --git a/backend/plugins/q_dev/impl/impl.go
b/backend/plugins/q_dev/impl/impl.go
index 9c3824924..80118212e 100644
--- a/backend/plugins/q_dev/impl/impl.go
+++ b/backend/plugins/q_dev/impl/impl.go
@@ -23,6 +23,7 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/q_dev/api"
@@ -39,6 +40,7 @@ var _ interface {
plugin.PluginModel
plugin.PluginSource
plugin.PluginMigration
+ plugin.DataSourcePluginBlueprintV200
plugin.CloseablePluginTask
} = (*QDev)(nil)
@@ -170,3 +172,10 @@ func (p QDev) Close(taskCtx plugin.TaskContext)
errors.Error {
data.S3Client.Close()
return nil
}
+
+func (p QDev) MakeDataSourcePipelinePlanV200(
+ connectionId uint64,
+ scopes []*coreModels.BlueprintScope,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+ return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(),
connectionId, scopes)
+}
diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go
b/backend/plugins/q_dev/models/migrationscripts/20251123_add_scope_config_id_to_s3_slice.go
similarity index 54%
copy from backend/plugins/q_dev/models/migrationscripts/register.go
copy to
backend/plugins/q_dev/models/migrationscripts/20251123_add_scope_config_id_to_s3_slice.go
index dfff79fae..e072e4f59 100644
--- a/backend/plugins/q_dev/models/migrationscripts/register.go
+++
b/backend/plugins/q_dev/models/migrationscripts/20251123_add_scope_config_id_to_s3_slice.go
@@ -18,16 +18,31 @@ limitations under the License.
package migrationscripts
import (
- "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
)
-// All return all migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(initTables),
- new(modifyFileMetaTable),
- new(addDisplayNameFields),
- new(addMissingMetrics),
- new(addS3SliceTable),
+type addScopeConfigIdToS3Slice struct{}
+
+func (*addScopeConfigIdToS3Slice) Up(basicRes context.BasicRes) errors.Error {
+ db := basicRes.GetDal()
+
+ // Add scope_config_id column to _tool_q_dev_s3_slices table
+ err := db.Exec(`
+ ALTER TABLE _tool_q_dev_s3_slices
+ ADD COLUMN scope_config_id BIGINT UNSIGNED DEFAULT 0
+ `)
+ if err != nil {
+ return errors.Convert(err)
}
+
+ return nil
+}
+
+func (*addScopeConfigIdToS3Slice) Version() uint64 {
+ return 20251123000001
+}
+
+func (*addScopeConfigIdToS3Slice) Name() string {
+ return "Add scope_config_id column to S3 slice table"
}
diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go
b/backend/plugins/q_dev/models/migrationscripts/register.go
index dfff79fae..427b3ac61 100644
--- a/backend/plugins/q_dev/models/migrationscripts/register.go
+++ b/backend/plugins/q_dev/models/migrationscripts/register.go
@@ -29,5 +29,6 @@ func All() []plugin.MigrationScript {
new(addDisplayNameFields),
new(addMissingMetrics),
new(addS3SliceTable),
+ new(addScopeConfigIdToS3Slice),
}
}
diff --git a/backend/plugins/q_dev/tasks/identity_client.go
b/backend/plugins/q_dev/tasks/identity_client.go
index 921d9abfe..855ce4ebd 100644
--- a/backend/plugins/q_dev/tasks/identity_client.go
+++ b/backend/plugins/q_dev/tasks/identity_client.go
@@ -70,6 +70,11 @@ func NewQDevIdentityClient(connection
*models.QDevConnection) (*QDevIdentityClie
// ResolveUserDisplayName resolves a user ID to a human-readable display name
// Returns the display name if found, otherwise returns the original userId as
fallback
func (client *QDevIdentityClient) ResolveUserDisplayName(userId string)
(string, error) {
+ // Check if client or IdentityStore is nil
+ if client == nil || client.IdentityStore == nil {
+ return userId, nil
+ }
+
input := &identitystore.DescribeUserInput{
IdentityStoreId: aws.String(client.StoreId),
UserId: aws.String(userId),
diff --git a/backend/plugins/q_dev/tasks/s3_data_extractor.go
b/backend/plugins/q_dev/tasks/s3_data_extractor.go
index 10ab6a7cd..8612ae03b 100644
--- a/backend/plugins/q_dev/tasks/s3_data_extractor.go
+++ b/backend/plugins/q_dev/tasks/s3_data_extractor.go
@@ -318,4 +318,5 @@ var ExtractQDevS3DataMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Extract data from S3 CSV files and save to database",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
+ Dependencies: []*plugin.SubTaskMeta{&CollectQDevS3FilesMeta},
}
diff --git a/backend/plugins/q_dev/tasks/s3_file_collector.go
b/backend/plugins/q_dev/tasks/s3_file_collector.go
index 37abf1845..e889d60db 100644
--- a/backend/plugins/q_dev/tasks/s3_file_collector.go
+++ b/backend/plugins/q_dev/tasks/s3_file_collector.go
@@ -114,4 +114,5 @@ var CollectQDevS3FilesMeta = plugin.SubTaskMeta{
EntryPoint: CollectQDevS3Files,
EnabledByDefault: true,
Description: "Collect S3 file metadata from AWS S3 bucket",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
}
diff --git
a/config-ui/src/plugins/register/q-dev/connection-fields/connection-test.tsx
b/config-ui/src/plugins/register/q-dev/connection-fields/connection-test.tsx
new file mode 100644
index 000000000..bc0eb7339
--- /dev/null
+++ b/config-ui/src/plugins/register/q-dev/connection-fields/connection-test.tsx
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { useState } from 'react';
+import { Button, Alert, Space } from 'antd';
+import { CheckCircleOutlined, ExclamationCircleOutlined, LoadingOutlined }
from '@ant-design/icons';
+
+import API from '@/api';
+import { operator } from '@/utils';
+
+interface Props {
+ plugin: string;
+ connectionId?: ID;
+ values: any;
+ initialValues: any;
+ disabled?: boolean;
+}
+
+interface TestResult {
+ success: boolean;
+ message: string;
+ details?: {
+ s3Access?: boolean;
+ identityCenterAccess?: boolean;
+ };
+}
+
+export const QDevConnectionTest = ({ plugin, connectionId, values,
initialValues, disabled }: Props) => {
+ const [testing, setTesting] = useState(false);
+ const [testResult, setTestResult] = useState<TestResult | null>(null);
+
+ const handleTest = async () => {
+ setTesting(true);
+ setTestResult(null);
+
+ try {
+ const [success, result] = await operator(
+ () => {
+ if (connectionId) {
+ // Test existing connection with only changed values
+ return API.connection.test(plugin, connectionId, {
+ authType: values.authType !== initialValues.authType ?
values.authType : undefined,
+ accessKeyId: values.accessKeyId !== initialValues.accessKeyId ?
values.accessKeyId : undefined,
+ secretAccessKey: values.secretAccessKey !==
initialValues.secretAccessKey ? values.secretAccessKey : undefined,
+ region: values.region !== initialValues.region ? values.region :
undefined,
+ bucket: values.bucket !== initialValues.bucket ? values.bucket :
undefined,
+ identityStoreId: values.identityStoreId !==
initialValues.identityStoreId ? values.identityStoreId : undefined,
+ identityStoreRegion: values.identityStoreRegion !==
initialValues.identityStoreRegion ? values.identityStoreRegion : undefined,
+ rateLimitPerHour: values.rateLimitPerHour !==
initialValues.rateLimitPerHour ? values.rateLimitPerHour : undefined,
+ proxy: values.proxy !== initialValues.proxy ? values.proxy :
undefined,
+ } as any);
+ } else {
+ // Test new connection with all values
+ return API.connection.testOld(plugin, {
+ authType: values.authType || 'access_key',
+ accessKeyId: values.accessKeyId || '',
+ secretAccessKey: values.secretAccessKey || '',
+ region: values.region || '',
+ bucket: values.bucket || '',
+ identityStoreId: values.identityStoreId || '',
+ identityStoreRegion: values.identityStoreRegion || '',
+ rateLimitPerHour: values.rateLimitPerHour || 20000,
+ proxy: values.proxy || '',
+ endpoint: '', // Not used by Q Developer
+ token: '', // Not used by Q Developer
+ } as any);
+ }
+ },
+ {
+ setOperating: () => {}, // We handle loading state ourselves
+ hideToast: true, // We show our own success/error messages
+ },
+ );
+
+ if (success && result) {
+ setTestResult({
+ success: true,
+ message: 'Connection test successful! AWS credentials and S3 access
verified.',
+ details: {
+ s3Access: true,
+ identityCenterAccess: values.identityStoreId ? true : undefined,
+ },
+ });
+ } else {
+ setTestResult({
+ success: false,
+ message: 'Connection test failed. Please check your configuration.',
+ });
+ }
+ } catch (error: any) {
+ let errorMessage = 'Connection test failed. Please check your
configuration.';
+
+ if (error?.response?.data?.message) {
+ errorMessage = error.response.data.message;
+ } else if (error?.message) {
+ errorMessage = error.message;
+ }
+
+ // Provide more specific error messages based on common issues
+ if (errorMessage.includes('InvalidAccessKeyId') ||
errorMessage.includes('SignatureDoesNotMatch')) {
+ errorMessage = 'Invalid AWS credentials. Please check your Access Key
ID and Secret Access Key.';
+ } else if (errorMessage.includes('NoSuchBucket')) {
+ errorMessage = 'S3 bucket not found. Please check the bucket name and
region.';
+ } else if (errorMessage.includes('AccessDenied')) {
+ errorMessage = 'Access denied. Please check your AWS permissions for
S3 and IAM Identity Center.';
+ } else if (errorMessage.includes('InvalidBucketName')) {
+ errorMessage = 'Invalid S3 bucket name. Please check the bucket name
format.';
+ } else if (errorMessage.includes('NoCredentialsError')) {
+ errorMessage = 'AWS credentials not found. Please provide valid Access
Key ID and Secret Access Key, or ensure IAM role is properly configured.';
+ }
+
+ setTestResult({
+ success: false,
+ message: errorMessage,
+ });
+ } finally {
+ setTesting(false);
+ }
+ };
+
+ const getAlertType = () => {
+ if (!testResult) return undefined;
+ return testResult.success ? 'success' : 'error';
+ };
+
+ const getAlertIcon = () => {
+ if (testing) return <LoadingOutlined />;
+ if (!testResult) return undefined;
+ return testResult.success ? <CheckCircleOutlined /> :
<ExclamationCircleOutlined />;
+ };
+
+ return (
+ <Space direction="vertical" style={{ width: '100%' }}>
+ <Button
+ type="default"
+ loading={testing}
+ disabled={disabled || testing}
+ onClick={handleTest}
+ style={{ marginTop: 16 }}
+ >
+ {testing ? 'Testing Connection...' : 'Test Connection'}
+ </Button>
+
+ {(testResult || testing) && (
+ <Alert
+ type={getAlertType()}
+ icon={getAlertIcon()}
+ message={testing ? 'Testing connection to AWS S3 and IAM Identity
Center...' : testResult?.message}
+ description={
+ testResult?.success && testResult.details ? (
+ <div>
+ <div>✓ S3 Access: Verified</div>
+ {testResult.details.identityCenterAccess && (
+ <div>✓ IAM Identity Center: Configured</div>
+ )}
+ {!values.identityStoreId && (
+ <div style={{ marginTop: 8, color: '#faad14' }}>
+ ⚠️ IAM Identity Center not configured - user display names
will show as user IDs
+ </div>
+ )}
+ </div>
+ ) : undefined
+ }
+ showIcon
+ style={{ marginTop: 8 }}
+ />
+ )}
+ </Space>
+ );
+};
\ No newline at end of file
