This is an automated email from the ASF dual-hosted git repository. warren pushed a commit to branch fix-clear-docker-images-before-build in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 0b86b3bda752f018de3e2db603ee01b9a6de6600 Author: warren <[email protected]> AuthorDate: Mon Jan 12 23:56:51 2026 +0800 feat: add scopes support for q_dev plugin --- backend/plugins/q_dev/api/blueprint_v200.go | 4 +- backend/plugins/q_dev/api/init.go | 7 ++- .../migrationscripts/register.go => api/scopes.go} | 25 ++++++----- backend/plugins/q_dev/impl/impl.go | 9 ++-- .../migrationscripts/20260112_add_scope_config.go | 49 ++++++++++++++++++++ .../q_dev/models/migrationscripts/register.go | 1 + .../register.go => scope_config.go} | 36 ++++++++++----- docker-compose-dev.yml | 52 +++++++++++----------- 8 files changed, 123 insertions(+), 60 deletions(-) diff --git a/backend/plugins/q_dev/api/blueprint_v200.go b/backend/plugins/q_dev/api/blueprint_v200.go index e3b845cb8..54e9dc070 100644 --- a/backend/plugins/q_dev/api/blueprint_v200.go +++ b/backend/plugins/q_dev/api/blueprint_v200.go @@ -56,7 +56,7 @@ func MakeDataSourcePipelinePlanV200( func makeDataSourcePipelinePlanV200( subtaskMetas []plugin.SubTaskMeta, - scopeDetails []*srvhelper.ScopeDetail[models.QDevS3Slice, srvhelper.NoScopeConfig], + scopeDetails []*srvhelper.ScopeDetail[models.QDevS3Slice, models.QDevScopeConfig], connection *models.QDevConnection, ) (coreModels.PipelinePlan, errors.Error) { plan := make(coreModels.PipelinePlan, len(scopeDetails)) @@ -86,7 +86,7 @@ func makeDataSourcePipelinePlanV200( } func makeScopesV200( - scopeDetails []*srvhelper.ScopeDetail[models.QDevS3Slice, srvhelper.NoScopeConfig], + scopeDetails []*srvhelper.ScopeDetail[models.QDevS3Slice, models.QDevScopeConfig], connection *models.QDevConnection, ) ([]plugin.Scope, errors.Error) { scopes := make([]plugin.Scope, 0) diff --git a/backend/plugins/q_dev/api/init.go b/backend/plugins/q_dev/api/init.go index 3bb67450b..3a3dc2f91 100644 --- a/backend/plugins/q_dev/api/init.go +++ b/backend/plugins/q_dev/api/init.go @@ -21,7 +21,6 @@ import ( "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/helpers/srvhelper" "github.com/apache/incubator-devlake/plugins/q_dev/models" "github.com/go-playground/validator/v10" ) @@ -29,7 +28,7 @@ import ( var vld *validator.Validate var connectionHelper *api.ConnectionApiHelper var basicRes context.BasicRes -var dsHelper *api.DsHelper[models.QDevConnection, models.QDevS3Slice, srvhelper.NoScopeConfig] +var dsHelper *api.DsHelper[models.QDevConnection, models.QDevS3Slice, models.QDevScopeConfig] func Init(br context.BasicRes, p plugin.PluginMeta) { basicRes = br @@ -41,13 +40,13 @@ func Init(br context.BasicRes, p plugin.PluginMeta) { ) dsHelper = api.NewDataSourceHelper[ - models.QDevConnection, models.QDevS3Slice, srvhelper.NoScopeConfig, + models.QDevConnection, models.QDevS3Slice, models.QDevScopeConfig, ]( basicRes, p.Name(), []string{"prefix", "basePath", "name"}, func(c models.QDevConnection) models.QDevConnection { return c.Sanitize() }, func(s models.QDevS3Slice) models.QDevS3Slice { return s.Sanitize() }, - nil, + func(sc models.QDevScopeConfig) models.QDevScopeConfig { return sc }, ) } diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go b/backend/plugins/q_dev/api/scopes.go similarity index 61% copy from backend/plugins/q_dev/models/migrationscripts/register.go copy to backend/plugins/q_dev/api/scopes.go index 86971e539..2b78a7b05 100644 --- a/backend/plugins/q_dev/models/migrationscripts/register.go +++ b/backend/plugins/q_dev/api/scopes.go @@ -15,21 +15,22 @@ See the License for the specific language governing permissions and limitations under the License. */ -package migrationscripts +package api import ( + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" ) -// All return all migration scripts -func All() []plugin.MigrationScript { - return []plugin.MigrationScript{ - new(initTables), - new(modifyFileMetaTable), - new(addDisplayNameFields), - new(addMissingMetrics), - new(addS3SliceTable), - new(addScopeConfigIdToS3Slice), - new(addScopeIdFields), - } +// GetScopes returns available S3 slices for a connection +func GetScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + // Use the existing GetScopeList function + return GetScopeList(input) +} + +// GetScopeConfigs returns empty list since we don't have scope configs yet +func GetScopeConfigs(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return &plugin.ApiResourceOutput{ + Body: []interface{}{}, + }, nil } diff --git a/backend/plugins/q_dev/impl/impl.go b/backend/plugins/q_dev/impl/impl.go index 80118212e..46bb166f5 100644 --- a/backend/plugins/q_dev/impl/impl.go +++ b/backend/plugins/q_dev/impl/impl.go @@ -57,6 +57,7 @@ func (p QDev) GetTablesInfo() []dal.Tabler { &models.QDevUserData{}, &models.QDevS3FileMeta{}, &models.QDevS3Slice{}, + &models.QDevScopeConfig{}, } } @@ -77,7 +78,7 @@ func (p QDev) Scope() plugin.ToolLayerScope { } func (p QDev) ScopeConfig() dal.Tabler { - return nil + return &models.QDevScopeConfig{} } func (p QDev) SubTaskMetas() []plugin.SubTaskMeta { @@ -150,7 +151,7 @@ func (p QDev) ApiResources() map[string]map[string]plugin.ApiResourceHandler { "POST": api.TestExistingConnection, }, "connections/:connectionId/scopes": { - "GET": api.GetScopeList, + "GET": api.GetScopes, "PUT": api.PutScopes, }, "connections/:connectionId/scopes/:scopeId": { @@ -158,8 +159,8 @@ func (p QDev) ApiResources() map[string]map[string]plugin.ApiResourceHandler { "PATCH": api.PatchScope, "DELETE": api.DeleteScope, }, - "connections/:connectionId/scopes/:scopeId/latest-sync-state": { - "GET": api.GetScopeLatestSyncState, + "connections/:connectionId/scope-configs": { + "GET": api.GetScopeConfigs, }, } } diff --git a/backend/plugins/q_dev/models/migrationscripts/20260112_add_scope_config.go b/backend/plugins/q_dev/models/migrationscripts/20260112_add_scope_config.go new file mode 100644 index 000000000..4d5965028 --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/20260112_add_scope_config.go @@ -0,0 +1,49 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addQDevScopeConfig struct{} + +type qdevScopeConfig20260112 struct { + archived.ScopeConfig + ProcessingEnabled bool `json:"processingEnabled" gorm:"type:boolean"` + FilePattern string `json:"filePattern" gorm:"type:varchar(255)"` +} + +func (qdevScopeConfig20260112) TableName() string { + return "_tool_q_dev_scope_configs" +} + +func (u *addQDevScopeConfig) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &qdevScopeConfig20260112{}) +} + +func (*addQDevScopeConfig) Version() uint64 { + return 20260112230000 +} + +func (*addQDevScopeConfig) Name() string { + return "Add Q Developer scope config table" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go b/backend/plugins/q_dev/models/migrationscripts/register.go index 86971e539..695ffed6b 100644 --- a/backend/plugins/q_dev/models/migrationscripts/register.go +++ b/backend/plugins/q_dev/models/migrationscripts/register.go @@ -31,5 +31,6 @@ func All() []plugin.MigrationScript { new(addS3SliceTable), new(addScopeConfigIdToS3Slice), new(addScopeIdFields), + new(addQDevScopeConfig), } } diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go b/backend/plugins/q_dev/models/scope_config.go similarity index 55% copy from backend/plugins/q_dev/models/migrationscripts/register.go copy to backend/plugins/q_dev/models/scope_config.go index 86971e539..f7da821b5 100644 --- a/backend/plugins/q_dev/models/migrationscripts/register.go +++ b/backend/plugins/q_dev/models/scope_config.go @@ -15,21 +15,33 @@ See the License for the specific language governing permissions and limitations under the License. */ -package migrationscripts +package models import ( + "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" ) -// All return all migration scripts -func All() []plugin.MigrationScript { - return []plugin.MigrationScript{ - new(initTables), - new(modifyFileMetaTable), - new(addDisplayNameFields), - new(addMissingMetrics), - new(addS3SliceTable), - new(addScopeConfigIdToS3Slice), - new(addScopeIdFields), - } +type QDevScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline"` + + // Processing options + ProcessingEnabled bool `json:"processingEnabled" mapstructure:"processingEnabled"` + + // File filtering + FilePattern string `json:"filePattern" mapstructure:"filePattern"` } + +func (QDevScopeConfig) TableName() string { + return "_tool_q_dev_scope_configs" +} + +func (c QDevScopeConfig) ScopeConfigId() uint64 { + return c.ID +} + +func (c QDevScopeConfig) ScopeConfigConnectionId() uint64 { + return c.ConnectionId +} + +var _ plugin.ToolLayerScopeConfig = (*QDevScopeConfig)(nil) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index b56720db2..52d14495c 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -32,31 +32,31 @@ services: --collation-server=utf8mb4_bin --skip-log-bin - postgres: - image: postgres:14.2 - volumes: - - postgres-storage:/var/lib/postgresql - restart: always - ports: - - 5432:5432 - environment: - POSTGRES_DB: lake - POSTGRES_USER: merico - POSTGRES_PASSWORD: merico - TZ: UTC + # postgres: + # image: postgres:14.2 + # volumes: + # - postgres-storage:/var/lib/postgresql + # restart: always + # ports: + # - 5432:5432 + # environment: + # POSTGRES_DB: lake + # POSTGRES_USER: merico + # POSTGRES_PASSWORD: merico + # TZ: UTC - postgres17: - image: postgres:17.2 - volumes: - - postgres17-storage:/var/lib/postgresql - restart: always - ports: - - 5432:5432 - environment: - POSTGRES_DB: lake - POSTGRES_USER: merico - POSTGRES_PASSWORD: merico - TZ: UTC + # postgres17: + # image: postgres:17.2 + # volumes: + # - postgres17-storage:/var/lib/postgresql + # restart: always + # ports: + # - 5432:5432 + # environment: + # POSTGRES_DB: lake + # POSTGRES_USER: merico + # POSTGRES_PASSWORD: merico + # TZ: UTC grafana: image: devlake.docker.scarf.sh/apache/devlake-dashboard:latest @@ -142,6 +142,6 @@ services: volumes: mysql-storage: grafana-storage: - postgres-storage: - postgres17-storage: + # postgres-storage: + # postgres17-storage: devlake-log: \ No newline at end of file
