This is an automated email from the ASF dual-hosted git repository.
Startrekzky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 79ef9f4fa chore: remove the dbt plugin and alpine-dbt docker image
(#8970) (#9084)
79ef9f4fa is described below
commit 79ef9f4fa1b513de03a0cc7776c23fd9d9fb6488
Author: Klesh Wong <[email protected]>
AuthorDate: Mon Aug 31 12:24:48 2026 +0800
chore: remove the dbt plugin and alpine-dbt docker image (#8970) (#9084)
Remove the deprecated dbt plugin from the backend, its alpine-dbt
Docker image, the config-ui example entry, and the dbt-mysql /
dbt-postgres Python dependencies. Update the plugin registration
test lists (table_info, schema_e2e, server_startup) accordingly.
---
backend/plugins/dbt/README.md | 18 --
backend/plugins/dbt/api/swagger.go | 65 ------
backend/plugins/dbt/dbt.go | 82 -------
backend/plugins/dbt/impl/impl.go | 91 --------
backend/plugins/dbt/tasks/convertor.go | 238 ---------------------
backend/plugins/dbt/tasks/git.go | 65 ------
backend/plugins/dbt/tasks/options.go | 132 ------------
backend/plugins/dbt/tasks/options_test.go | 110 ----------
backend/plugins/dbt/tasks/task_data.go | 49 -----
.../plugins/schema_e2e/migration_schema_test.go | 2 -
backend/plugins/table_info_test.go | 2 -
backend/python/requirements.txt | 4 +-
backend/test/e2e/services/server_startup_test.go | 2 -
.../components/advanced-editor/example/dbt.ts | 50 -----
.../components/advanced-editor/example/index.ts | 6 -
devops/docker/alpine-dbt/Dockerfile | 28 ---
devops/docker/alpine-dbt/README.md | 27 ---
17 files changed, 1 insertion(+), 970 deletions(-)
diff --git a/backend/plugins/dbt/README.md b/backend/plugins/dbt/README.md
deleted file mode 100644
index ced495606..000000000
--- a/backend/plugins/dbt/README.md
+++ /dev/null
@@ -1,18 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-Please see details in the [Apache DevLake
website](https://devlake.apache.org/docs/Plugins/dbt)
\ No newline at end of file
diff --git a/backend/plugins/dbt/api/swagger.go
b/backend/plugins/dbt/api/swagger.go
deleted file mode 100644
index a0e800af6..000000000
--- a/backend/plugins/dbt/api/swagger.go
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-// @Summary blueprints plan for dbt
-// @Description blueprints plan for dbt
-// @Tags plugins/dbt
-// @Accept application/json
-// @Param blueprint body DbtBlueprintPlan true "json"
-// @Router /blueprints/dbt/blueprint-plan [post]
-func _() {}
-
-type Options struct {
- ProjectPath string `json:"projectPath"`
- ProjectGitURL string `json:"projectGitURL"`
- ProjectName string `json:"projectName"`
- ProjectTarget string `json:"projectTarget"`
- SelectedModels []string `json:"selectedModels"`
- Args []string `json:"args"`
- FailFast bool `json:"failFast"`
- ProfilesPath string `json:"profilesPath"`
- Profile string `json:"profile"`
- Threads int `json:"threads"`
- NoVersionCheck bool `json:"noVersionCheck"`
- ExcludeModels []string `json:"excludeModels"`
- Selector string `json:"selector"`
- State string `json:"state"`
- Defer bool `json:"defer"`
- NoDefer bool `json:"noDefer"`
- FullRefresh bool `json:"fullRefresh"`
- ProjectVars struct {
- Demokey1 string `json:"demokey1"`
- Demokey2 string `json:"demokey2"`
- } `json:"projectVars"`
-}
-type Plan struct {
- Plugin string `json:"plugin"`
- Options Options `json:"options"`
-}
-type DbtBlueprintPlan [][]Plan
-
-// @Summary pipelines plan for dbt
-// @Description pipelines plan for dbt
-// @Tags plugins/dbt
-// @Accept application/json
-// @Param pipeline body DbtPipelinePlan true "json"
-// @Router /pipelines/dbt/pipeline-plan [post]
-func _() {}
-
-type DbtPipelinePlan [][]Plan
diff --git a/backend/plugins/dbt/dbt.go b/backend/plugins/dbt/dbt.go
deleted file mode 100644
index 02d055a9d..000000000
--- a/backend/plugins/dbt/dbt.go
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "github.com/apache/incubator-devlake/core/runner"
- "github.com/apache/incubator-devlake/plugins/dbt/impl"
- "github.com/spf13/cobra"
-)
-
-var PluginEntry impl.Dbt
-
-// standalone mode for debugging
-func main() {
- dbtCmd := &cobra.Command{Use: "dbt"}
- projectPath := dbtCmd.Flags().StringP("projectPath", "p",
"/Users/abeizn/demoapp", "user dbt project directory.")
- projectGitURL := dbtCmd.Flags().StringP("projectGitURL", "g", "", "user
dbt project git url.")
- projectName := dbtCmd.Flags().StringP("projectName", "n", "demoapp",
"user dbt project name.")
- projectTarget := dbtCmd.Flags().StringP("projectTarget", "o", "dev",
"this is the default target your dbt project will use.")
- modelsSlice := []string{"my_first_dbt_model", "my_second_dbt_model"}
- selectedModels := dbtCmd.Flags().StringSliceP("models", "m",
modelsSlice, "dbt select models")
- failFast := dbtCmd.Flags().BoolP("failFast", "", false, "dbt fail fast")
- profilesPath := dbtCmd.Flags().StringP("profilesPath", "",
"/Users/abeizn/.dbt", "dbt profiles path")
- profile := dbtCmd.Flags().StringP("profile", "", "default", "dbt
profile")
- threads := dbtCmd.Flags().IntP("threads", "", 1, "dbt threads")
- noVersionCheck := dbtCmd.Flags().BoolP("noVersionCheck", "", false,
"dbt no version check")
- excludeModels := dbtCmd.Flags().StringSliceP("excludeModels", "",
[]string{}, "dbt exclude models")
- selector := dbtCmd.Flags().StringP("selector", "", "", "dbt selector")
- state := dbtCmd.Flags().StringP("state", "", "", "dbt state")
- deferFlag := dbtCmd.Flags().BoolP("defer", "", false, "dbt defer")
- noDefer := dbtCmd.Flags().BoolP("noDefer", "", false, "dbt no defer")
- fullRefresh := dbtCmd.Flags().BoolP("fullRefresh", "", false, "dbt full
refresh")
- dbtArgs := dbtCmd.Flags().StringSliceP("args", "a", []string{}, "dbt
run args")
- projectVars := make(map[string]string)
- projectVars["event_min_id"] = "7581"
- projectVars["event_max_id"] = "7582"
- dbtCmd.Flags().StringToStringVarP(&projectVars, "projectVars", "v",
projectVars, "dbt provides variables to provide data to models for
compilation.")
- timeAfter := dbtCmd.Flags().StringP("timeAfter", "a", "", "collect data
that are created after specified time, ie 2006-01-02T15:04:05Z")
-
- dbtCmd.Run = func(cmd *cobra.Command, args []string) {
- projectVarsConvert := make(map[string]interface{},
len(projectVars))
- for k, v := range projectVars {
- projectVarsConvert[k] = v
- }
- runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- "projectPath": *projectPath,
- "projectName": *projectName,
- "projectTarget": *projectTarget,
- "selectedModels": *selectedModels,
- "projectVars": projectVarsConvert,
- "projectGitURL": *projectGitURL,
- "args": dbtArgs,
- "failFast": *failFast,
- "profilesPath": *profilesPath,
- "profile": *profile,
- "threads": *threads,
- "noVersionCheck": *noVersionCheck,
- "excludeModels": *excludeModels,
- "selector": *selector,
- "state": *state,
- "defer": *deferFlag,
- "noDefer": *noDefer,
- "fullRefresh": *fullRefresh,
- }, *timeAfter)
- }
- runner.RunCmd(dbtCmd)
-}
diff --git a/backend/plugins/dbt/impl/impl.go b/backend/plugins/dbt/impl/impl.go
deleted file mode 100644
index c984187ca..000000000
--- a/backend/plugins/dbt/impl/impl.go
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package impl
-
-import (
- "fmt"
-
- "github.com/apache/incubator-devlake/core/dal"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/dbt/tasks"
-)
-
-var _ interface {
- plugin.PluginMeta
- plugin.PluginTask
- plugin.CloseablePluginTask
- plugin.PluginModel
-} = (*Dbt)(nil)
-
-type Dbt struct{}
-
-func (p Dbt) Description() string {
- return "Convert data by dbt"
-}
-
-func (p Dbt) SubTaskMetas() []plugin.SubTaskMeta {
- return []plugin.SubTaskMeta{
- tasks.GitMeta,
- tasks.DbtConverterMeta,
- }
-}
-
-func (p Dbt) GetTablesInfo() []dal.Tabler {
- return []dal.Tabler{}
-}
-
-func (p Dbt) PrepareTaskData(taskCtx plugin.TaskContext, options
map[string]interface{}) (interface{}, errors.Error) {
- taskCtx.GetLogger().Warn(nil, "The dbt plugin is deprecated and will be
removed on August 31, 2026. Please migrate to alternative transformation
approaches.")
- var op tasks.DbtOptions
- err := helper.Decode(options, &op, nil)
- if err != nil {
- return nil, err
- }
- if err := tasks.PrepareOptions(&op,
taskCtx.GetConfig(tasks.DbtProjectBaseDirConfigKey)); err != nil {
- return nil, err
- }
-
- if op.ProjectTarget == "" {
- op.ProjectTarget = "dev"
- }
-
- return &tasks.DbtTaskData{
- Options: &op,
- }, nil
-}
-
-func (p Dbt) RootPkgPath() string {
- return "github.com/apache/incubator-devlake/plugins/dbt"
-}
-
-func (p Dbt) Close(taskCtx plugin.TaskContext) errors.Error {
- data, ok := taskCtx.GetData().(*tasks.DbtTaskData)
- if !ok || data == nil || data.Options == nil ||
!data.Options.ManagedProjectDir {
- return nil
- }
- if err := tasks.CleanupManagedProjectDir(data.Options); err != nil {
- return errors.Default.Wrap(err, fmt.Sprintf("cleanup dbt
project path %q", data.Options.ProjectPath))
- }
- return nil
-}
-
-func (p Dbt) Name() string {
- return "dbt"
-}
diff --git a/backend/plugins/dbt/tasks/convertor.go
b/backend/plugins/dbt/tasks/convertor.go
deleted file mode 100644
index a3a2d3f56..000000000
--- a/backend/plugins/dbt/tasks/convertor.go
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
- "bufio"
- "encoding/json"
- "net"
- "net/url"
- "os"
- "os/exec"
- "path/filepath"
- "strconv"
- "strings"
-
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/spf13/viper"
-)
-
-func DbtConverter(taskCtx plugin.SubTaskContext) (err errors.Error) {
- logger := taskCtx.GetLogger()
- taskCtx.SetProgress(0, -1)
- data := taskCtx.GetData().(*DbtTaskData)
- models := data.Options.SelectedModels
- projectPath := data.Options.ProjectPath
- projectName := data.Options.ProjectName
- projectTarget := data.Options.ProjectTarget
- projectVars := data.Options.ProjectVars
- args := data.Options.Args
- failFast := data.Options.FailFast
- threads := data.Options.Threads
- noVersionCheck := data.Options.NoVersionCheck
- excludeModels := data.Options.ExcludeModels
- selector := data.Options.Selector
- state := data.Options.State
- deferFlag := data.Options.Defer
- noDefer := data.Options.NoDefer
- fullRefresh := data.Options.FullRefresh
- profilesPath := data.Options.ProfilesPath
- profile := data.Options.Profile
-
- defaultProfilesPath := filepath.Join(projectPath, "profiles.yml")
- _, err = errors.Convert01(os.Stat(defaultProfilesPath))
- // if profiles.yml not exist, create it manually
- if err != nil {
- dbUrl := taskCtx.GetConfig("DB_URL")
- u, err := errors.Convert01(url.Parse(dbUrl))
- if err != nil {
- return err
- }
- dbType := u.Scheme
- dbUsername := u.User.Username()
- dbPassword, _ := u.User.Password()
- dbServer, dbPort, _ := net.SplitHostPort(u.Host)
- dbDataBase := u.Path[1:]
- var dbSchema string
- flag := strings.Compare(dbType, "mysql")
- if flag == 0 {
- // mysql database
- dbSchema = dbDataBase
- } else {
- // other database
- mapQuery, err :=
errors.Convert01(url.ParseQuery(u.RawQuery))
- if err != nil {
- return err
- }
- if value, ok := mapQuery["search_path"]; ok {
- if len(value) < 1 {
- return errors.Default.New("DB_URL
search_path parses error")
- }
- dbSchema = value[0]
- } else {
- dbSchema = "public"
- }
- }
- config := viper.New()
- config.Set(projectName+".target", projectTarget)
- config.Set(projectName+".outputs."+projectTarget+".type",
dbType)
- dbPortInt, _ := strconv.Atoi(dbPort)
- config.Set(projectName+".outputs."+projectTarget+".port",
dbPortInt)
- config.Set(projectName+".outputs."+projectTarget+".password",
dbPassword)
- config.Set(projectName+".outputs."+projectTarget+".schema",
dbSchema)
- if flag == 0 {
-
config.Set(projectName+".outputs."+projectTarget+".server", dbServer)
-
config.Set(projectName+".outputs."+projectTarget+".username", dbUsername)
-
config.Set(projectName+".outputs."+projectTarget+".database", dbDataBase)
- } else {
-
config.Set(projectName+".outputs."+projectTarget+".host", dbServer)
-
config.Set(projectName+".outputs."+projectTarget+".user", dbUsername)
-
config.Set(projectName+".outputs."+projectTarget+".dbname", dbDataBase)
- }
- err = errors.Convert(config.WriteConfigAs(defaultProfilesPath))
- if err != nil {
- return err
- }
- }
- // if package.yml exist, install dbt dependencies
- defaultPackagesPath := filepath.Join(projectPath, "packages.yml")
- _, err = errors.Convert01(os.Stat(defaultPackagesPath))
- if err == nil {
- cmd := exec.Command("dbt", "deps")
- err = errors.Convert(cmd.Start())
- if err != nil {
- return err
- }
- }
-
- dbtExecParams := []string{"dbt", "run", "--project-dir", projectPath}
- if projectVars != nil {
- jsonProjectVars, err := json.Marshal(projectVars)
- if err != nil {
- return errors.Default.New("parameters vars json marshal
error")
- }
- dbtExecParams = append(dbtExecParams, "--vars")
- dbtExecParams = append(dbtExecParams, string(jsonProjectVars))
- }
- if len(models) > 0 {
- dbtExecParams = append(dbtExecParams, "--select")
- dbtExecParams = append(dbtExecParams, models...)
- }
- if args != nil {
- dbtExecParams = append(dbtExecParams, args...)
- }
- if failFast {
- dbtExecParams = append(dbtExecParams, "--fail-fast")
- }
- if threads != 0 {
- dbtExecParams = append(dbtExecParams, "--threads")
- dbtExecParams = append(dbtExecParams, strconv.Itoa(threads))
- }
- if noVersionCheck {
- dbtExecParams = append(dbtExecParams, "--no-version-check")
- }
- if excludeModels != nil {
- dbtExecParams = append(dbtExecParams, "--exclude")
- dbtExecParams = append(dbtExecParams, excludeModels...)
- }
- if selector != "" {
- dbtExecParams = append(dbtExecParams, "--selector")
- dbtExecParams = append(dbtExecParams, selector)
- }
- if state != "" {
- dbtExecParams = append(dbtExecParams, "--state")
- dbtExecParams = append(dbtExecParams, state)
- }
- if deferFlag {
- dbtExecParams = append(dbtExecParams, "--defer")
- }
- if noDefer {
- dbtExecParams = append(dbtExecParams, "--no-defer")
- }
- if fullRefresh {
- dbtExecParams = append(dbtExecParams, "--full-refresh")
- }
- if profilesPath != "" {
- dbtExecParams = append(dbtExecParams, "--profiles-dir")
- dbtExecParams = append(dbtExecParams, profilesPath)
- } else {
- // default projectPath
- dbtExecParams = append(dbtExecParams, "--profiles-dir")
- dbtExecParams = append(dbtExecParams, projectPath)
- }
- if profile != "" {
- dbtExecParams = append(dbtExecParams, "--profile")
- dbtExecParams = append(dbtExecParams, profile)
- }
- cmd := exec.Command(dbtExecParams[0], dbtExecParams[1:]...)
- logger.Info("dbt run script: ", cmd)
-
- stdout, stdoutErr := cmd.StdoutPipe()
- if stdoutErr != nil {
- return errors.Convert(stdoutErr)
- }
-
- if err = errors.Convert(cmd.Start()); err != nil {
- return err
- }
-
- // prevent zombie process
- var errStr string
- defer func() {
- err = errors.Convert(cmd.Wait())
- if err != nil {
- logger.Error(err, "The DBT project run failed!")
- err = errors.SubtaskErr.New(errStr)
- } else {
- logger.Info("The DBT project run ended.")
- }
- }()
-
- scanner := bufio.NewScanner(stdout)
- for scanner.Scan() {
- line := scanner.Text()
- logger.Info(line)
- if strings.Contains(line, "Encountered an error") || errStr !=
"" {
- errStr += line + "\n"
- }
- if strings.Contains(line, "of") && strings.Contains(line, "OK")
{
- taskCtx.IncProgress(1)
- }
- }
- if err = errors.Convert(scanner.Err()); err != nil {
- logger.Error(err, "dbt read stdout failed.")
- return err
- }
-
- // close stdout
- if closeErr := stdout.Close(); closeErr != nil && err == nil {
- logger.Error(closeErr, "dbt close stdout failed.")
- return errors.Convert(closeErr)
- }
-
- return err
-}
-
-var DbtConverterMeta = plugin.SubTaskMeta{
- Name: "DbtConverter",
- EntryPoint: DbtConverter,
- EnabledByDefault: true,
- Description: "Convert data by dbt",
- Dependencies: []*plugin.SubTaskMeta{&GitMeta},
-}
diff --git a/backend/plugins/dbt/tasks/git.go b/backend/plugins/dbt/tasks/git.go
deleted file mode 100644
index b43f3efcf..000000000
--- a/backend/plugins/dbt/tasks/git.go
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
- "os"
- "os/exec"
-
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
-)
-
-func Git(taskCtx plugin.SubTaskContext) errors.Error {
- logger := taskCtx.GetLogger()
- data := taskCtx.GetData().(*DbtTaskData)
- if data.Options.ProjectGitURL == "" {
- return nil
- }
-
- projectBaseDir, err := ensureProjectBaseDir(data.Options.ProjectBaseDir)
- if err != nil {
- logger.Error(err, "prepare dbt workspace failed")
- return err
- }
- projectPath, mkErr := os.MkdirTemp(projectBaseDir, "project-*")
- if mkErr != nil {
- logger.Error(mkErr, "create managed dbt project directory
failed")
- return errors.Convert(mkErr)
- }
- data.Options.ProjectPath = projectPath
-
- // git clone from ProjectGitURL into a managed temporary project
directory
- cmd := exec.Command("git", "clone", data.Options.ProjectGitURL,
data.Options.ProjectPath)
- logger.Info("start clone dbt project: %v", cmd)
- out, cloneErr := cmd.CombinedOutput()
- if cloneErr != nil {
- _ = os.RemoveAll(data.Options.ProjectPath)
- logger.Error(cloneErr, "clone dbt project failed")
- return errors.Convert(cloneErr)
- }
- logger.Info("clone dbt project success: %v", string(out))
- return nil
-}
-
-var GitMeta = plugin.SubTaskMeta{
- Name: "Git",
- EntryPoint: Git,
- EnabledByDefault: true,
- Description: "Clone dbt project from git",
-}
diff --git a/backend/plugins/dbt/tasks/options.go
b/backend/plugins/dbt/tasks/options.go
deleted file mode 100644
index 496290d43..000000000
--- a/backend/plugins/dbt/tasks/options.go
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
- "net/url"
- "os"
- "path/filepath"
- "strings"
-
- "github.com/apache/incubator-devlake/core/errors"
-)
-
-const (
- DbtProjectBaseDirConfigKey = "DBT_PROJECTS_DIR"
- dbtProjectBaseDirName = "devlake-dbt-projects"
-)
-
-func PrepareOptions(op *DbtOptions, configuredBaseDir string) errors.Error {
- if op == nil {
- return errors.Default.New("dbt options are required")
- }
-
- baseDir, err := normalizeBaseDir(configuredBaseDir)
- if err != nil {
- return err
- }
- op.ProjectBaseDir = baseDir
- op.ProjectPath = strings.TrimSpace(op.ProjectPath)
- op.ProjectGitURL = strings.TrimSpace(op.ProjectGitURL)
-
- if op.ProjectGitURL != "" {
- if err := validateProjectGitURL(op.ProjectGitURL); err != nil {
- return err
- }
- op.ProjectPath = ""
- op.ManagedProjectDir = true
- return nil
- }
-
- if op.ProjectPath == "" {
- return errors.Default.New("projectPath is required for local
dbt projects")
- }
-
- projectPath, err := normalizePathWithinBase(baseDir, op.ProjectPath)
- if err != nil {
- return err
- }
- op.ProjectPath = projectPath
- return nil
-}
-
-func normalizeBaseDir(configuredBaseDir string) (string, errors.Error) {
- baseDir := strings.TrimSpace(configuredBaseDir)
- if baseDir == "" {
- baseDir = filepath.Join(os.TempDir(), dbtProjectBaseDirName)
- }
- baseDir, err := filepath.Abs(filepath.Clean(baseDir))
- if err != nil {
- return "", errors.Convert(err)
- }
- return baseDir, nil
-}
-
-func normalizePathWithinBase(baseDir string, candidate string) (string,
errors.Error) {
- normalizedPath, err := filepath.Abs(filepath.Clean(candidate))
- if err != nil {
- return "", errors.Convert(err)
- }
- rel, err := filepath.Rel(baseDir, normalizedPath)
- if err != nil {
- return "", errors.Convert(err)
- }
- if rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator))
{
- return "", errors.Default.New("projectPath must stay within " +
baseDir)
- }
- return normalizedPath, nil
-}
-
-func validateProjectGitURL(rawURL string) errors.Error {
- u, err := url.Parse(rawURL)
- if err != nil {
- return errors.Convert(err)
- }
- if u.Scheme != "https" && u.Scheme != "ssh" {
- return errors.Default.New("projectGitURL must use https:// or
ssh://")
- }
- if u.Host == "" {
- return errors.Default.New("projectGitURL must include a
hostname")
- }
- if u.Path == "" || u.Path == "/" {
- return errors.Default.New("projectGitURL must include a
repository path")
- }
- return nil
-}
-
-func ensureProjectBaseDir(baseDir string) (string, errors.Error) {
- baseDir, err := normalizeBaseDir(baseDir)
- if err != nil {
- return "", err
- }
- if err := os.MkdirAll(baseDir, 0o755); err != nil {
- return "", errors.Convert(err)
- }
- return baseDir, nil
-}
-
-func CleanupManagedProjectDir(op *DbtOptions) errors.Error {
- if op == nil || !op.ManagedProjectDir || op.ProjectPath == "" {
- return nil
- }
- projectPath, err := normalizePathWithinBase(op.ProjectBaseDir,
op.ProjectPath)
- if err != nil {
- return err
- }
- return errors.Convert(os.RemoveAll(projectPath))
-}
diff --git a/backend/plugins/dbt/tasks/options_test.go
b/backend/plugins/dbt/tasks/options_test.go
deleted file mode 100644
index 740e35273..000000000
--- a/backend/plugins/dbt/tasks/options_test.go
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
- "os"
- "path/filepath"
- "testing"
-)
-
-func TestPrepareOptionsNormalizesLocalProjectPath(t *testing.T) {
- baseDir := t.TempDir()
- projectDir := filepath.Join(baseDir, "nested", "..", "project")
-
- op := &DbtOptions{ProjectPath: projectDir}
- if err := PrepareOptions(op, baseDir); err != nil {
- t.Fatalf("PrepareOptions returned error: %v", err)
- }
-
- want := filepath.Join(baseDir, "project")
- if op.ProjectPath != want {
- t.Fatalf("ProjectPath = %q, want %q", op.ProjectPath, want)
- }
- if op.ProjectBaseDir != baseDir {
- t.Fatalf("ProjectBaseDir = %q, want %q", op.ProjectBaseDir,
baseDir)
- }
- if op.ManagedProjectDir {
- t.Fatal("ManagedProjectDir should be false for local projects")
- }
-}
-
-func TestPrepareOptionsRejectsPathTraversal(t *testing.T) {
- baseDir := t.TempDir()
- op := &DbtOptions{ProjectPath: filepath.Join(baseDir, "..", "outside")}
-
- if err := PrepareOptions(op, baseDir); err == nil {
- t.Fatal("PrepareOptions should reject projectPath outside the
configured base directory")
- }
-}
-
-func TestPrepareOptionsRejectsUnsafeGitURLs(t *testing.T) {
- baseDir := t.TempDir()
- cases := []string{
- "file:///tmp/evil",
- "../relative/repo.git",
- "[email protected]:apache/incubator-devlake.git",
- }
-
- for _, rawURL := range cases {
- t.Run(rawURL, func(t *testing.T) {
- op := &DbtOptions{ProjectGitURL: rawURL}
- if err := PrepareOptions(op, baseDir); err == nil {
- t.Fatalf("PrepareOptions should reject %q",
rawURL)
- }
- })
- }
-}
-
-func TestPrepareOptionsAllowsManagedGitClone(t *testing.T) {
- baseDir := t.TempDir()
- op := &DbtOptions{
- ProjectPath: filepath.Join(baseDir, "ignored"),
- ProjectGitURL:
"https://github.com/apache/incubator-devlake.git";,
- }
-
- if err := PrepareOptions(op, baseDir); err != nil {
- t.Fatalf("PrepareOptions returned error: %v", err)
- }
- if !op.ManagedProjectDir {
- t.Fatal("ManagedProjectDir should be true for git-backed
projects")
- }
- if op.ProjectPath != "" {
- t.Fatalf("ProjectPath = %q, want empty for managed git clones",
op.ProjectPath)
- }
-}
-
-func TestCleanupManagedProjectDirRemovesManagedPath(t *testing.T) {
- baseDir := t.TempDir()
- projectDir := filepath.Join(baseDir, "clone")
- if err := os.MkdirAll(projectDir, 0o755); err != nil {
- t.Fatalf("MkdirAll returned error: %v", err)
- }
-
- op := &DbtOptions{
- ProjectBaseDir: baseDir,
- ProjectPath: projectDir,
- ManagedProjectDir: true,
- }
- if err := CleanupManagedProjectDir(op); err != nil {
- t.Fatalf("CleanupManagedProjectDir returned error: %v", err)
- }
- if _, err := os.Stat(projectDir); !os.IsNotExist(err) {
- t.Fatalf("expected %q to be removed, stat err = %v",
projectDir, err)
- }
-}
diff --git a/backend/plugins/dbt/tasks/task_data.go
b/backend/plugins/dbt/tasks/task_data.go
deleted file mode 100644
index 00555fa2b..000000000
--- a/backend/plugins/dbt/tasks/task_data.go
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-type DbtOptions struct {
- ProjectPath string `json:"projectPath"`
- ProjectName string `json:"projectName"`
- ProjectTarget string `json:"projectTarget"`
- // clone from git to projectPath if projectGitURL is not empty
- ProjectGitURL string `json:"projectGitURL"`
- ProjectVars map[string]interface{} `json:"projectVars"`
- SelectedModels []string `json:"selectedModels"`
- FailFast bool `json:"failFast"`
- ProfilesPath string `json:"profilesPath"`
- Profile string `json:"profile"`
- Threads int `json:"threads"`
- NoVersionCheck bool `json:"noVersionCheck"`
- ExcludeModels []string `json:"excludeModels"`
- Selector string `json:"selector"`
- State string `json:"state"`
- Defer bool `json:"defer"`
- NoDefer bool `json:"noDefer"`
- FullRefresh bool `json:"fullRefresh"`
- // deprecated, dbt run args
- Args []string `json:"args"`
- Tasks []string `json:"tasks,omitempty"`
-
- ProjectBaseDir string `json:"-"`
- ManagedProjectDir bool `json:"-"`
-}
-
-type DbtTaskData struct {
- Options *DbtOptions
-}
diff --git a/backend/plugins/schema_e2e/migration_schema_test.go
b/backend/plugins/schema_e2e/migration_schema_test.go
index 80b397520..c51788aaa 100644
--- a/backend/plugins/schema_e2e/migration_schema_test.go
+++ b/backend/plugins/schema_e2e/migration_schema_test.go
@@ -56,7 +56,6 @@ import (
claudeCode
"github.com/apache/incubator-devlake/plugins/claude_code/impl"
clickup "github.com/apache/incubator-devlake/plugins/clickup/impl"
customize "github.com/apache/incubator-devlake/plugins/customize/impl"
- dbt "github.com/apache/incubator-devlake/plugins/dbt/impl"
dora "github.com/apache/incubator-devlake/plugins/dora/impl"
feishu "github.com/apache/incubator-devlake/plugins/feishu/impl"
copilot "github.com/apache/incubator-devlake/plugins/gh-copilot/impl"
@@ -107,7 +106,6 @@ func allGoPlugins() []plugin.PluginMeta {
claudeCode.ClaudeCode{},
clickup.ClickUp{},
customize.Customize{},
- dbt.Dbt{},
dora.Dora{},
feishu.Feishu{},
copilot.GhCopilot{},
diff --git a/backend/plugins/table_info_test.go
b/backend/plugins/table_info_test.go
index f4182fe9d..2c79feaf2 100644
--- a/backend/plugins/table_info_test.go
+++ b/backend/plugins/table_info_test.go
@@ -32,7 +32,6 @@ import (
claudeCode
"github.com/apache/incubator-devlake/plugins/claude_code/impl"
clickup "github.com/apache/incubator-devlake/plugins/clickup/impl"
customize "github.com/apache/incubator-devlake/plugins/customize/impl"
- dbt "github.com/apache/incubator-devlake/plugins/dbt/impl"
dora "github.com/apache/incubator-devlake/plugins/dora/impl"
feishu "github.com/apache/incubator-devlake/plugins/feishu/impl"
copilot "github.com/apache/incubator-devlake/plugins/gh-copilot/impl"
@@ -80,7 +79,6 @@ func Test_GetPluginTablesInfo(t *testing.T) {
checker.FeedIn("argocd/models", argocd.ArgoCD{}.GetTablesInfo)
checker.FeedIn("asana/models", asana.Asana{}.GetTablesInfo)
checker.FeedIn("customize/models", customize.Customize{}.GetTablesInfo)
- checker.FeedIn("dbt", dbt.Dbt{}.GetTablesInfo)
checker.FeedIn("dora/models", dora.Dora{}.GetTablesInfo)
checker.FeedIn("feishu/models", feishu.Feishu{}.GetTablesInfo)
checker.FeedIn("gitee/models", gitee.Gitee{}.GetTablesInfo)
diff --git a/backend/python/requirements.txt b/backend/python/requirements.txt
index fe3fc3fd6..87cb77788 100644
--- a/backend/python/requirements.txt
+++ b/backend/python/requirements.txt
@@ -1,5 +1,3 @@
# runtime dependencies
setuptools
-wheel
-dbt-mysql
-dbt-postgres
\ No newline at end of file
+wheel
\ No newline at end of file
diff --git a/backend/test/e2e/services/server_startup_test.go
b/backend/test/e2e/services/server_startup_test.go
index f52dafeb0..b8565136d 100644
--- a/backend/test/e2e/services/server_startup_test.go
+++ b/backend/test/e2e/services/server_startup_test.go
@@ -25,7 +25,6 @@ import (
bamboo "github.com/apache/incubator-devlake/plugins/bamboo/impl"
bitbucket "github.com/apache/incubator-devlake/plugins/bitbucket/impl"
customize "github.com/apache/incubator-devlake/plugins/customize/impl"
- dbt "github.com/apache/incubator-devlake/plugins/dbt/impl"
dora "github.com/apache/incubator-devlake/plugins/dora/impl"
feishu "github.com/apache/incubator-devlake/plugins/feishu/impl"
gitee "github.com/apache/incubator-devlake/plugins/gitee/impl"
@@ -66,7 +65,6 @@ func loadGoPlugins() []plugin.PluginMeta {
bamboo.Bamboo{},
bitbucket.Bitbucket{},
customize.Customize{},
- dbt.Dbt{},
dora.Dora{},
feishu.Feishu{},
gitee.Gitee{},
diff --git
a/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/dbt.ts
b/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/dbt.ts
deleted file mode 100644
index c09a3a228..000000000
---
a/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/dbt.ts
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-const dbt = [
- [
- {
- plugin: 'dbt',
- options: {
- projectPath: '/tmp/devlake-dbt-projects/my-project',
- projectGitURL: '',
- projectName: 'myproject',
- projectTarget: 'dev',
- selectedModels: ['model_one', 'model_two'],
- projectVars: {
- demokey1: 'demovalue1',
- demokey2: 'demovalue2',
- },
- failFast: false,
- profilesPath: '',
- profile: '',
- threads: 0,
- noVersionCheck: false,
- excludeModels: [],
- selector: '',
- state: '',
- defer: false,
- noDefer: false,
- fullRefresh: false,
- args: [],
- },
- },
- ],
-];
-
-export default dbt;
diff --git
a/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/index.ts
b/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/index.ts
index 7da6da3ca..756a59e3b 100644
---
a/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/index.ts
+++
b/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/index.ts
@@ -17,7 +17,6 @@
*/
import customize from './customize';
-import dbt from './dbt';
import feishu from './feishu';
import general from './general';
import gitextractor from './gitextractor';
@@ -71,11 +70,6 @@ export const EXAMPLE_CONFIG = [
name: 'Load Feishu Configuration',
config: feishu,
},
- {
- id: 'dbt',
- name: 'Load DBT Configuration',
- config: dbt,
- },
{
id: 'tapd',
name: 'Load TAPD Configuration',
diff --git a/devops/docker/alpine-dbt/Dockerfile
b/devops/docker/alpine-dbt/Dockerfile
deleted file mode 100644
index 0f8a773ce..000000000
--- a/devops/docker/alpine-dbt/Dockerfile
+++ /dev/null
@@ -1,28 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# current tag: mericodev/alpine-dbt:0.0.1
-FROM --platform=linux/amd64 alpine:3.15
-RUN apk add --no-cache musl-dev libgit2-dev libffi-dev \
- && apk add --no-cache gcc
-
-ENV PYTHONUNBUFFERED=1
-RUN apk add --update --no-cache python3-dev && ln -sf python3 /usr/bin/python
-RUN python3 -m ensurepip
-RUN pip3 install --no-cache --upgrade pip setuptools
-RUN pip3 install dbt-mysql
-RUN pip3 install dbt-postgres
-RUN apk add --no-cache tar
diff --git a/devops/docker/alpine-dbt/README.md
b/devops/docker/alpine-dbt/README.md
deleted file mode 100644
index 41b663a63..000000000
--- a/devops/docker/alpine-dbt/README.md
+++ /dev/null
@@ -1,27 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-# alpine-dbt
-alpine linux image with dbt installed
-
-https://hub.docker.com/r/mericodev/alpine-dbt
-
-## release
-```shell
-export VERSION=0.0.1
-docker build -t mericodev/alpine-dbt:$VERSION .
-docker push mericodev/alpine-dbt:$VERSION
-```
