This is an automated email from the ASF dual-hosted git repository. klesh pushed a commit to branch kw-rm-dbt in repository https://gitbox.apache.org/repos/asf/devlake.git
commit 25ae22677b9a3ef4c86f2c932258ec88f2b523a8 Author: Klesh Wong <[email protected]> AuthorDate: Mon Aug 31 12:09:04 2026 +0800 chore: remove the dbt plugin and alpine-dbt docker image (#8970) Remove the deprecated dbt plugin from the backend, its alpine-dbt Docker image, the config-ui example entry, and the dbt-mysql / dbt-postgres Python dependencies. Update the plugin registration test lists (table_info, schema_e2e, server_startup) accordingly. --- backend/plugins/dbt/README.md | 18 -- backend/plugins/dbt/api/swagger.go | 65 ------ backend/plugins/dbt/dbt.go | 82 ------- backend/plugins/dbt/impl/impl.go | 91 -------- backend/plugins/dbt/tasks/convertor.go | 238 --------------------- backend/plugins/dbt/tasks/git.go | 65 ------ backend/plugins/dbt/tasks/options.go | 132 ------------ backend/plugins/dbt/tasks/options_test.go | 110 ---------- backend/plugins/dbt/tasks/task_data.go | 49 ----- .../plugins/schema_e2e/migration_schema_test.go | 2 - backend/plugins/table_info_test.go | 2 - backend/python/requirements.txt | 4 +- backend/test/e2e/services/server_startup_test.go | 2 - .../components/advanced-editor/example/dbt.ts | 50 ----- .../components/advanced-editor/example/index.ts | 6 - devops/docker/alpine-dbt/Dockerfile | 28 --- devops/docker/alpine-dbt/README.md | 27 --- 17 files changed, 1 insertion(+), 970 deletions(-) diff --git a/backend/plugins/dbt/README.md b/backend/plugins/dbt/README.md deleted file mode 100644 index ced495606..000000000 --- a/backend/plugins/dbt/README.md +++ /dev/null @@ -1,18 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> - -Please see details in the [Apache DevLake website](https://devlake.apache.org/docs/Plugins/dbt) \ No newline at end of file diff --git a/backend/plugins/dbt/api/swagger.go b/backend/plugins/dbt/api/swagger.go deleted file mode 100644 index a0e800af6..000000000 --- a/backend/plugins/dbt/api/swagger.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package api - -// @Summary blueprints plan for dbt -// @Description blueprints plan for dbt -// @Tags plugins/dbt -// @Accept application/json -// @Param blueprint body DbtBlueprintPlan true "json" -// @Router /blueprints/dbt/blueprint-plan [post] -func _() {} - -type Options struct { - ProjectPath string `json:"projectPath"` - ProjectGitURL string `json:"projectGitURL"` - ProjectName string `json:"projectName"` - ProjectTarget string `json:"projectTarget"` - SelectedModels []string `json:"selectedModels"` - Args []string `json:"args"` - FailFast bool `json:"failFast"` - ProfilesPath string `json:"profilesPath"` - Profile string `json:"profile"` - Threads int `json:"threads"` - NoVersionCheck bool `json:"noVersionCheck"` - ExcludeModels []string `json:"excludeModels"` - Selector string `json:"selector"` - State string `json:"state"` - Defer bool `json:"defer"` - NoDefer bool `json:"noDefer"` - FullRefresh bool `json:"fullRefresh"` - ProjectVars struct { - Demokey1 string `json:"demokey1"` - Demokey2 string `json:"demokey2"` - } `json:"projectVars"` -} -type Plan struct { - Plugin string `json:"plugin"` - Options Options `json:"options"` -} -type DbtBlueprintPlan [][]Plan - -// @Summary pipelines plan for dbt -// @Description pipelines plan for dbt -// @Tags plugins/dbt -// @Accept application/json -// @Param pipeline body DbtPipelinePlan true "json" -// @Router /pipelines/dbt/pipeline-plan [post] -func _() {} - -type DbtPipelinePlan [][]Plan diff --git a/backend/plugins/dbt/dbt.go b/backend/plugins/dbt/dbt.go deleted file mode 100644 index 02d055a9d..000000000 --- a/backend/plugins/dbt/dbt.go +++ /dev/null @@ -1,82 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "github.com/apache/incubator-devlake/core/runner" - "github.com/apache/incubator-devlake/plugins/dbt/impl" - "github.com/spf13/cobra" -) - -var PluginEntry impl.Dbt - -// standalone mode for debugging -func main() { - dbtCmd := &cobra.Command{Use: "dbt"} - projectPath := dbtCmd.Flags().StringP("projectPath", "p", "/Users/abeizn/demoapp", "user dbt project directory.") - projectGitURL := dbtCmd.Flags().StringP("projectGitURL", "g", "", "user dbt project git url.") - projectName := dbtCmd.Flags().StringP("projectName", "n", "demoapp", "user dbt project name.") - projectTarget := dbtCmd.Flags().StringP("projectTarget", "o", "dev", "this is the default target your dbt project will use.") - modelsSlice := []string{"my_first_dbt_model", "my_second_dbt_model"} - selectedModels := dbtCmd.Flags().StringSliceP("models", "m", modelsSlice, "dbt select models") - failFast := dbtCmd.Flags().BoolP("failFast", "", false, "dbt fail fast") - profilesPath := dbtCmd.Flags().StringP("profilesPath", "", "/Users/abeizn/.dbt", "dbt profiles path") - profile := dbtCmd.Flags().StringP("profile", "", "default", "dbt profile") - threads := dbtCmd.Flags().IntP("threads", "", 1, "dbt threads") - noVersionCheck := dbtCmd.Flags().BoolP("noVersionCheck", "", false, "dbt no version check") - excludeModels := dbtCmd.Flags().StringSliceP("excludeModels", "", []string{}, "dbt exclude models") - selector := dbtCmd.Flags().StringP("selector", "", "", "dbt selector") - state := dbtCmd.Flags().StringP("state", "", "", "dbt state") - deferFlag := dbtCmd.Flags().BoolP("defer", "", false, "dbt defer") - noDefer := dbtCmd.Flags().BoolP("noDefer", "", false, "dbt no defer") - fullRefresh := dbtCmd.Flags().BoolP("fullRefresh", "", false, "dbt full refresh") - dbtArgs := dbtCmd.Flags().StringSliceP("args", "a", []string{}, "dbt run args") - projectVars := make(map[string]string) - projectVars["event_min_id"] = "7581" - projectVars["event_max_id"] = "7582" - dbtCmd.Flags().StringToStringVarP(&projectVars, "projectVars", "v", projectVars, "dbt provides variables to provide data to models for compilation.") - timeAfter := dbtCmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") - - dbtCmd.Run = func(cmd *cobra.Command, args []string) { - projectVarsConvert := make(map[string]interface{}, len(projectVars)) - for k, v := range projectVars { - projectVarsConvert[k] = v - } - runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ - "projectPath": *projectPath, - "projectName": *projectName, - "projectTarget": *projectTarget, - "selectedModels": *selectedModels, - "projectVars": projectVarsConvert, - "projectGitURL": *projectGitURL, - "args": dbtArgs, - "failFast": *failFast, - "profilesPath": *profilesPath, - "profile": *profile, - "threads": *threads, - "noVersionCheck": *noVersionCheck, - "excludeModels": *excludeModels, - "selector": *selector, - "state": *state, - "defer": *deferFlag, - "noDefer": *noDefer, - "fullRefresh": *fullRefresh, - }, *timeAfter) - } - runner.RunCmd(dbtCmd) -} diff --git a/backend/plugins/dbt/impl/impl.go b/backend/plugins/dbt/impl/impl.go deleted file mode 100644 index c984187ca..000000000 --- a/backend/plugins/dbt/impl/impl.go +++ /dev/null @@ -1,91 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package impl - -import ( - "fmt" - - "github.com/apache/incubator-devlake/core/dal" - "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/plugin" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/dbt/tasks" -) - -var _ interface { - plugin.PluginMeta - plugin.PluginTask - plugin.CloseablePluginTask - plugin.PluginModel -} = (*Dbt)(nil) - -type Dbt struct{} - -func (p Dbt) Description() string { - return "Convert data by dbt" -} - -func (p Dbt) SubTaskMetas() []plugin.SubTaskMeta { - return []plugin.SubTaskMeta{ - tasks.GitMeta, - tasks.DbtConverterMeta, - } -} - -func (p Dbt) GetTablesInfo() []dal.Tabler { - return []dal.Tabler{} -} - -func (p Dbt) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { - taskCtx.GetLogger().Warn(nil, "The dbt plugin is deprecated and will be removed on August 31, 2026. Please migrate to alternative transformation approaches.") - var op tasks.DbtOptions - err := helper.Decode(options, &op, nil) - if err != nil { - return nil, err - } - if err := tasks.PrepareOptions(&op, taskCtx.GetConfig(tasks.DbtProjectBaseDirConfigKey)); err != nil { - return nil, err - } - - if op.ProjectTarget == "" { - op.ProjectTarget = "dev" - } - - return &tasks.DbtTaskData{ - Options: &op, - }, nil -} - -func (p Dbt) RootPkgPath() string { - return "github.com/apache/incubator-devlake/plugins/dbt" -} - -func (p Dbt) Close(taskCtx plugin.TaskContext) errors.Error { - data, ok := taskCtx.GetData().(*tasks.DbtTaskData) - if !ok || data == nil || data.Options == nil || !data.Options.ManagedProjectDir { - return nil - } - if err := tasks.CleanupManagedProjectDir(data.Options); err != nil { - return errors.Default.Wrap(err, fmt.Sprintf("cleanup dbt project path %q", data.Options.ProjectPath)) - } - return nil -} - -func (p Dbt) Name() string { - return "dbt" -} diff --git a/backend/plugins/dbt/tasks/convertor.go b/backend/plugins/dbt/tasks/convertor.go deleted file mode 100644 index a3a2d3f56..000000000 --- a/backend/plugins/dbt/tasks/convertor.go +++ /dev/null @@ -1,238 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tasks - -import ( - "bufio" - "encoding/json" - "net" - "net/url" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - - "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/plugin" - "github.com/spf13/viper" -) - -func DbtConverter(taskCtx plugin.SubTaskContext) (err errors.Error) { - logger := taskCtx.GetLogger() - taskCtx.SetProgress(0, -1) - data := taskCtx.GetData().(*DbtTaskData) - models := data.Options.SelectedModels - projectPath := data.Options.ProjectPath - projectName := data.Options.ProjectName - projectTarget := data.Options.ProjectTarget - projectVars := data.Options.ProjectVars - args := data.Options.Args - failFast := data.Options.FailFast - threads := data.Options.Threads - noVersionCheck := data.Options.NoVersionCheck - excludeModels := data.Options.ExcludeModels - selector := data.Options.Selector - state := data.Options.State - deferFlag := data.Options.Defer - noDefer := data.Options.NoDefer - fullRefresh := data.Options.FullRefresh - profilesPath := data.Options.ProfilesPath - profile := data.Options.Profile - - defaultProfilesPath := filepath.Join(projectPath, "profiles.yml") - _, err = errors.Convert01(os.Stat(defaultProfilesPath)) - // if profiles.yml not exist, create it manually - if err != nil { - dbUrl := taskCtx.GetConfig("DB_URL") - u, err := errors.Convert01(url.Parse(dbUrl)) - if err != nil { - return err - } - dbType := u.Scheme - dbUsername := u.User.Username() - dbPassword, _ := u.User.Password() - dbServer, dbPort, _ := net.SplitHostPort(u.Host) - dbDataBase := u.Path[1:] - var dbSchema string - flag := strings.Compare(dbType, "mysql") - if flag == 0 { - // mysql database - dbSchema = dbDataBase - } else { - // other database - mapQuery, err := errors.Convert01(url.ParseQuery(u.RawQuery)) - if err != nil { - return err - } - if value, ok := mapQuery["search_path"]; ok { - if len(value) < 1 { - return errors.Default.New("DB_URL search_path parses error") - } - dbSchema = value[0] - } else { - dbSchema = "public" - } - } - config := viper.New() - config.Set(projectName+".target", projectTarget) - config.Set(projectName+".outputs."+projectTarget+".type", dbType) - dbPortInt, _ := strconv.Atoi(dbPort) - config.Set(projectName+".outputs."+projectTarget+".port", dbPortInt) - config.Set(projectName+".outputs."+projectTarget+".password", dbPassword) - config.Set(projectName+".outputs."+projectTarget+".schema", dbSchema) - if flag == 0 { - config.Set(projectName+".outputs."+projectTarget+".server", dbServer) - config.Set(projectName+".outputs."+projectTarget+".username", dbUsername) - config.Set(projectName+".outputs."+projectTarget+".database", dbDataBase) - } else { - config.Set(projectName+".outputs."+projectTarget+".host", dbServer) - config.Set(projectName+".outputs."+projectTarget+".user", dbUsername) - config.Set(projectName+".outputs."+projectTarget+".dbname", dbDataBase) - } - err = errors.Convert(config.WriteConfigAs(defaultProfilesPath)) - if err != nil { - return err - } - } - // if package.yml exist, install dbt dependencies - defaultPackagesPath := filepath.Join(projectPath, "packages.yml") - _, err = errors.Convert01(os.Stat(defaultPackagesPath)) - if err == nil { - cmd := exec.Command("dbt", "deps") - err = errors.Convert(cmd.Start()) - if err != nil { - return err - } - } - - dbtExecParams := []string{"dbt", "run", "--project-dir", projectPath} - if projectVars != nil { - jsonProjectVars, err := json.Marshal(projectVars) - if err != nil { - return errors.Default.New("parameters vars json marshal error") - } - dbtExecParams = append(dbtExecParams, "--vars") - dbtExecParams = append(dbtExecParams, string(jsonProjectVars)) - } - if len(models) > 0 { - dbtExecParams = append(dbtExecParams, "--select") - dbtExecParams = append(dbtExecParams, models...) - } - if args != nil { - dbtExecParams = append(dbtExecParams, args...) - } - if failFast { - dbtExecParams = append(dbtExecParams, "--fail-fast") - } - if threads != 0 { - dbtExecParams = append(dbtExecParams, "--threads") - dbtExecParams = append(dbtExecParams, strconv.Itoa(threads)) - } - if noVersionCheck { - dbtExecParams = append(dbtExecParams, "--no-version-check") - } - if excludeModels != nil { - dbtExecParams = append(dbtExecParams, "--exclude") - dbtExecParams = append(dbtExecParams, excludeModels...) - } - if selector != "" { - dbtExecParams = append(dbtExecParams, "--selector") - dbtExecParams = append(dbtExecParams, selector) - } - if state != "" { - dbtExecParams = append(dbtExecParams, "--state") - dbtExecParams = append(dbtExecParams, state) - } - if deferFlag { - dbtExecParams = append(dbtExecParams, "--defer") - } - if noDefer { - dbtExecParams = append(dbtExecParams, "--no-defer") - } - if fullRefresh { - dbtExecParams = append(dbtExecParams, "--full-refresh") - } - if profilesPath != "" { - dbtExecParams = append(dbtExecParams, "--profiles-dir") - dbtExecParams = append(dbtExecParams, profilesPath) - } else { - // default projectPath - dbtExecParams = append(dbtExecParams, "--profiles-dir") - dbtExecParams = append(dbtExecParams, projectPath) - } - if profile != "" { - dbtExecParams = append(dbtExecParams, "--profile") - dbtExecParams = append(dbtExecParams, profile) - } - cmd := exec.Command(dbtExecParams[0], dbtExecParams[1:]...) - logger.Info("dbt run script: ", cmd) - - stdout, stdoutErr := cmd.StdoutPipe() - if stdoutErr != nil { - return errors.Convert(stdoutErr) - } - - if err = errors.Convert(cmd.Start()); err != nil { - return err - } - - // prevent zombie process - var errStr string - defer func() { - err = errors.Convert(cmd.Wait()) - if err != nil { - logger.Error(err, "The DBT project run failed!") - err = errors.SubtaskErr.New(errStr) - } else { - logger.Info("The DBT project run ended.") - } - }() - - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - line := scanner.Text() - logger.Info(line) - if strings.Contains(line, "Encountered an error") || errStr != "" { - errStr += line + "\n" - } - if strings.Contains(line, "of") && strings.Contains(line, "OK") { - taskCtx.IncProgress(1) - } - } - if err = errors.Convert(scanner.Err()); err != nil { - logger.Error(err, "dbt read stdout failed.") - return err - } - - // close stdout - if closeErr := stdout.Close(); closeErr != nil && err == nil { - logger.Error(closeErr, "dbt close stdout failed.") - return errors.Convert(closeErr) - } - - return err -} - -var DbtConverterMeta = plugin.SubTaskMeta{ - Name: "DbtConverter", - EntryPoint: DbtConverter, - EnabledByDefault: true, - Description: "Convert data by dbt", - Dependencies: []*plugin.SubTaskMeta{&GitMeta}, -} diff --git a/backend/plugins/dbt/tasks/git.go b/backend/plugins/dbt/tasks/git.go deleted file mode 100644 index b43f3efcf..000000000 --- a/backend/plugins/dbt/tasks/git.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tasks - -import ( - "os" - "os/exec" - - "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/plugin" -) - -func Git(taskCtx plugin.SubTaskContext) errors.Error { - logger := taskCtx.GetLogger() - data := taskCtx.GetData().(*DbtTaskData) - if data.Options.ProjectGitURL == "" { - return nil - } - - projectBaseDir, err := ensureProjectBaseDir(data.Options.ProjectBaseDir) - if err != nil { - logger.Error(err, "prepare dbt workspace failed") - return err - } - projectPath, mkErr := os.MkdirTemp(projectBaseDir, "project-*") - if mkErr != nil { - logger.Error(mkErr, "create managed dbt project directory failed") - return errors.Convert(mkErr) - } - data.Options.ProjectPath = projectPath - - // git clone from ProjectGitURL into a managed temporary project directory - cmd := exec.Command("git", "clone", data.Options.ProjectGitURL, data.Options.ProjectPath) - logger.Info("start clone dbt project: %v", cmd) - out, cloneErr := cmd.CombinedOutput() - if cloneErr != nil { - _ = os.RemoveAll(data.Options.ProjectPath) - logger.Error(cloneErr, "clone dbt project failed") - return errors.Convert(cloneErr) - } - logger.Info("clone dbt project success: %v", string(out)) - return nil -} - -var GitMeta = plugin.SubTaskMeta{ - Name: "Git", - EntryPoint: Git, - EnabledByDefault: true, - Description: "Clone dbt project from git", -} diff --git a/backend/plugins/dbt/tasks/options.go b/backend/plugins/dbt/tasks/options.go deleted file mode 100644 index 496290d43..000000000 --- a/backend/plugins/dbt/tasks/options.go +++ /dev/null @@ -1,132 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tasks - -import ( - "net/url" - "os" - "path/filepath" - "strings" - - "github.com/apache/incubator-devlake/core/errors" -) - -const ( - DbtProjectBaseDirConfigKey = "DBT_PROJECTS_DIR" - dbtProjectBaseDirName = "devlake-dbt-projects" -) - -func PrepareOptions(op *DbtOptions, configuredBaseDir string) errors.Error { - if op == nil { - return errors.Default.New("dbt options are required") - } - - baseDir, err := normalizeBaseDir(configuredBaseDir) - if err != nil { - return err - } - op.ProjectBaseDir = baseDir - op.ProjectPath = strings.TrimSpace(op.ProjectPath) - op.ProjectGitURL = strings.TrimSpace(op.ProjectGitURL) - - if op.ProjectGitURL != "" { - if err := validateProjectGitURL(op.ProjectGitURL); err != nil { - return err - } - op.ProjectPath = "" - op.ManagedProjectDir = true - return nil - } - - if op.ProjectPath == "" { - return errors.Default.New("projectPath is required for local dbt projects") - } - - projectPath, err := normalizePathWithinBase(baseDir, op.ProjectPath) - if err != nil { - return err - } - op.ProjectPath = projectPath - return nil -} - -func normalizeBaseDir(configuredBaseDir string) (string, errors.Error) { - baseDir := strings.TrimSpace(configuredBaseDir) - if baseDir == "" { - baseDir = filepath.Join(os.TempDir(), dbtProjectBaseDirName) - } - baseDir, err := filepath.Abs(filepath.Clean(baseDir)) - if err != nil { - return "", errors.Convert(err) - } - return baseDir, nil -} - -func normalizePathWithinBase(baseDir string, candidate string) (string, errors.Error) { - normalizedPath, err := filepath.Abs(filepath.Clean(candidate)) - if err != nil { - return "", errors.Convert(err) - } - rel, err := filepath.Rel(baseDir, normalizedPath) - if err != nil { - return "", errors.Convert(err) - } - if rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { - return "", errors.Default.New("projectPath must stay within " + baseDir) - } - return normalizedPath, nil -} - -func validateProjectGitURL(rawURL string) errors.Error { - u, err := url.Parse(rawURL) - if err != nil { - return errors.Convert(err) - } - if u.Scheme != "https" && u.Scheme != "ssh" { - return errors.Default.New("projectGitURL must use https:// or ssh://") - } - if u.Host == "" { - return errors.Default.New("projectGitURL must include a hostname") - } - if u.Path == "" || u.Path == "/" { - return errors.Default.New("projectGitURL must include a repository path") - } - return nil -} - -func ensureProjectBaseDir(baseDir string) (string, errors.Error) { - baseDir, err := normalizeBaseDir(baseDir) - if err != nil { - return "", err - } - if err := os.MkdirAll(baseDir, 0o755); err != nil { - return "", errors.Convert(err) - } - return baseDir, nil -} - -func CleanupManagedProjectDir(op *DbtOptions) errors.Error { - if op == nil || !op.ManagedProjectDir || op.ProjectPath == "" { - return nil - } - projectPath, err := normalizePathWithinBase(op.ProjectBaseDir, op.ProjectPath) - if err != nil { - return err - } - return errors.Convert(os.RemoveAll(projectPath)) -} diff --git a/backend/plugins/dbt/tasks/options_test.go b/backend/plugins/dbt/tasks/options_test.go deleted file mode 100644 index 740e35273..000000000 --- a/backend/plugins/dbt/tasks/options_test.go +++ /dev/null @@ -1,110 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tasks - -import ( - "os" - "path/filepath" - "testing" -) - -func TestPrepareOptionsNormalizesLocalProjectPath(t *testing.T) { - baseDir := t.TempDir() - projectDir := filepath.Join(baseDir, "nested", "..", "project") - - op := &DbtOptions{ProjectPath: projectDir} - if err := PrepareOptions(op, baseDir); err != nil { - t.Fatalf("PrepareOptions returned error: %v", err) - } - - want := filepath.Join(baseDir, "project") - if op.ProjectPath != want { - t.Fatalf("ProjectPath = %q, want %q", op.ProjectPath, want) - } - if op.ProjectBaseDir != baseDir { - t.Fatalf("ProjectBaseDir = %q, want %q", op.ProjectBaseDir, baseDir) - } - if op.ManagedProjectDir { - t.Fatal("ManagedProjectDir should be false for local projects") - } -} - -func TestPrepareOptionsRejectsPathTraversal(t *testing.T) { - baseDir := t.TempDir() - op := &DbtOptions{ProjectPath: filepath.Join(baseDir, "..", "outside")} - - if err := PrepareOptions(op, baseDir); err == nil { - t.Fatal("PrepareOptions should reject projectPath outside the configured base directory") - } -} - -func TestPrepareOptionsRejectsUnsafeGitURLs(t *testing.T) { - baseDir := t.TempDir() - cases := []string{ - "file:///tmp/evil", - "../relative/repo.git", - "[email protected]:apache/incubator-devlake.git", - } - - for _, rawURL := range cases { - t.Run(rawURL, func(t *testing.T) { - op := &DbtOptions{ProjectGitURL: rawURL} - if err := PrepareOptions(op, baseDir); err == nil { - t.Fatalf("PrepareOptions should reject %q", rawURL) - } - }) - } -} - -func TestPrepareOptionsAllowsManagedGitClone(t *testing.T) { - baseDir := t.TempDir() - op := &DbtOptions{ - ProjectPath: filepath.Join(baseDir, "ignored"), - ProjectGitURL: "https://github.com/apache/incubator-devlake.git";, - } - - if err := PrepareOptions(op, baseDir); err != nil { - t.Fatalf("PrepareOptions returned error: %v", err) - } - if !op.ManagedProjectDir { - t.Fatal("ManagedProjectDir should be true for git-backed projects") - } - if op.ProjectPath != "" { - t.Fatalf("ProjectPath = %q, want empty for managed git clones", op.ProjectPath) - } -} - -func TestCleanupManagedProjectDirRemovesManagedPath(t *testing.T) { - baseDir := t.TempDir() - projectDir := filepath.Join(baseDir, "clone") - if err := os.MkdirAll(projectDir, 0o755); err != nil { - t.Fatalf("MkdirAll returned error: %v", err) - } - - op := &DbtOptions{ - ProjectBaseDir: baseDir, - ProjectPath: projectDir, - ManagedProjectDir: true, - } - if err := CleanupManagedProjectDir(op); err != nil { - t.Fatalf("CleanupManagedProjectDir returned error: %v", err) - } - if _, err := os.Stat(projectDir); !os.IsNotExist(err) { - t.Fatalf("expected %q to be removed, stat err = %v", projectDir, err) - } -} diff --git a/backend/plugins/dbt/tasks/task_data.go b/backend/plugins/dbt/tasks/task_data.go deleted file mode 100644 index 00555fa2b..000000000 --- a/backend/plugins/dbt/tasks/task_data.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tasks - -type DbtOptions struct { - ProjectPath string `json:"projectPath"` - ProjectName string `json:"projectName"` - ProjectTarget string `json:"projectTarget"` - // clone from git to projectPath if projectGitURL is not empty - ProjectGitURL string `json:"projectGitURL"` - ProjectVars map[string]interface{} `json:"projectVars"` - SelectedModels []string `json:"selectedModels"` - FailFast bool `json:"failFast"` - ProfilesPath string `json:"profilesPath"` - Profile string `json:"profile"` - Threads int `json:"threads"` - NoVersionCheck bool `json:"noVersionCheck"` - ExcludeModels []string `json:"excludeModels"` - Selector string `json:"selector"` - State string `json:"state"` - Defer bool `json:"defer"` - NoDefer bool `json:"noDefer"` - FullRefresh bool `json:"fullRefresh"` - // deprecated, dbt run args - Args []string `json:"args"` - Tasks []string `json:"tasks,omitempty"` - - ProjectBaseDir string `json:"-"` - ManagedProjectDir bool `json:"-"` -} - -type DbtTaskData struct { - Options *DbtOptions -} diff --git a/backend/plugins/schema_e2e/migration_schema_test.go b/backend/plugins/schema_e2e/migration_schema_test.go index 80b397520..c51788aaa 100644 --- a/backend/plugins/schema_e2e/migration_schema_test.go +++ b/backend/plugins/schema_e2e/migration_schema_test.go @@ -56,7 +56,6 @@ import ( claudeCode "github.com/apache/incubator-devlake/plugins/claude_code/impl" clickup "github.com/apache/incubator-devlake/plugins/clickup/impl" customize "github.com/apache/incubator-devlake/plugins/customize/impl" - dbt "github.com/apache/incubator-devlake/plugins/dbt/impl" dora "github.com/apache/incubator-devlake/plugins/dora/impl" feishu "github.com/apache/incubator-devlake/plugins/feishu/impl" copilot "github.com/apache/incubator-devlake/plugins/gh-copilot/impl" @@ -107,7 +106,6 @@ func allGoPlugins() []plugin.PluginMeta { claudeCode.ClaudeCode{}, clickup.ClickUp{}, customize.Customize{}, - dbt.Dbt{}, dora.Dora{}, feishu.Feishu{}, copilot.GhCopilot{}, diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index f4182fe9d..2c79feaf2 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -32,7 +32,6 @@ import ( claudeCode "github.com/apache/incubator-devlake/plugins/claude_code/impl" clickup "github.com/apache/incubator-devlake/plugins/clickup/impl" customize "github.com/apache/incubator-devlake/plugins/customize/impl" - dbt "github.com/apache/incubator-devlake/plugins/dbt/impl" dora "github.com/apache/incubator-devlake/plugins/dora/impl" feishu "github.com/apache/incubator-devlake/plugins/feishu/impl" copilot "github.com/apache/incubator-devlake/plugins/gh-copilot/impl" @@ -80,7 +79,6 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("argocd/models", argocd.ArgoCD{}.GetTablesInfo) checker.FeedIn("asana/models", asana.Asana{}.GetTablesInfo) checker.FeedIn("customize/models", customize.Customize{}.GetTablesInfo) - checker.FeedIn("dbt", dbt.Dbt{}.GetTablesInfo) checker.FeedIn("dora/models", dora.Dora{}.GetTablesInfo) checker.FeedIn("feishu/models", feishu.Feishu{}.GetTablesInfo) checker.FeedIn("gitee/models", gitee.Gitee{}.GetTablesInfo) diff --git a/backend/python/requirements.txt b/backend/python/requirements.txt index fe3fc3fd6..87cb77788 100644 --- a/backend/python/requirements.txt +++ b/backend/python/requirements.txt @@ -1,5 +1,3 @@ # runtime dependencies setuptools -wheel -dbt-mysql -dbt-postgres \ No newline at end of file +wheel \ No newline at end of file diff --git a/backend/test/e2e/services/server_startup_test.go b/backend/test/e2e/services/server_startup_test.go index f52dafeb0..b8565136d 100644 --- a/backend/test/e2e/services/server_startup_test.go +++ b/backend/test/e2e/services/server_startup_test.go @@ -25,7 +25,6 @@ import ( bamboo "github.com/apache/incubator-devlake/plugins/bamboo/impl" bitbucket "github.com/apache/incubator-devlake/plugins/bitbucket/impl" customize "github.com/apache/incubator-devlake/plugins/customize/impl" - dbt "github.com/apache/incubator-devlake/plugins/dbt/impl" dora "github.com/apache/incubator-devlake/plugins/dora/impl" feishu "github.com/apache/incubator-devlake/plugins/feishu/impl" gitee "github.com/apache/incubator-devlake/plugins/gitee/impl" @@ -66,7 +65,6 @@ func loadGoPlugins() []plugin.PluginMeta { bamboo.Bamboo{}, bitbucket.Bitbucket{}, customize.Customize{}, - dbt.Dbt{}, dora.Dora{}, feishu.Feishu{}, gitee.Gitee{}, diff --git a/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/dbt.ts b/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/dbt.ts deleted file mode 100644 index c09a3a228..000000000 --- a/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/dbt.ts +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -const dbt = [ - [ - { - plugin: 'dbt', - options: { - projectPath: '/tmp/devlake-dbt-projects/my-project', - projectGitURL: '', - projectName: 'myproject', - projectTarget: 'dev', - selectedModels: ['model_one', 'model_two'], - projectVars: { - demokey1: 'demovalue1', - demokey2: 'demovalue2', - }, - failFast: false, - profilesPath: '', - profile: '', - threads: 0, - noVersionCheck: false, - excludeModels: [], - selector: '', - state: '', - defer: false, - noDefer: false, - fullRefresh: false, - args: [], - }, - }, - ], -]; - -export default dbt; diff --git a/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/index.ts b/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/index.ts index 7da6da3ca..756a59e3b 100644 --- a/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/index.ts +++ b/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/index.ts @@ -17,7 +17,6 @@ */ import customize from './customize'; -import dbt from './dbt'; import feishu from './feishu'; import general from './general'; import gitextractor from './gitextractor'; @@ -71,11 +70,6 @@ export const EXAMPLE_CONFIG = [ name: 'Load Feishu Configuration', config: feishu, }, - { - id: 'dbt', - name: 'Load DBT Configuration', - config: dbt, - }, { id: 'tapd', name: 'Load TAPD Configuration', diff --git a/devops/docker/alpine-dbt/Dockerfile b/devops/docker/alpine-dbt/Dockerfile deleted file mode 100644 index 0f8a773ce..000000000 --- a/devops/docker/alpine-dbt/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# current tag: mericodev/alpine-dbt:0.0.1 -FROM --platform=linux/amd64 alpine:3.15 -RUN apk add --no-cache musl-dev libgit2-dev libffi-dev \ - && apk add --no-cache gcc - -ENV PYTHONUNBUFFERED=1 -RUN apk add --update --no-cache python3-dev && ln -sf python3 /usr/bin/python -RUN python3 -m ensurepip -RUN pip3 install --no-cache --upgrade pip setuptools -RUN pip3 install dbt-mysql -RUN pip3 install dbt-postgres -RUN apk add --no-cache tar diff --git a/devops/docker/alpine-dbt/README.md b/devops/docker/alpine-dbt/README.md deleted file mode 100644 index 41b663a63..000000000 --- a/devops/docker/alpine-dbt/README.md +++ /dev/null @@ -1,27 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> -# alpine-dbt -alpine linux image with dbt installed - -https://hub.docker.com/r/mericodev/alpine-dbt - -## release -```shell -export VERSION=0.0.1 -docker build -t mericodev/alpine-dbt:$VERSION . -docker push mericodev/alpine-dbt:$VERSION -```
