[incubator-devlake] branch release-v0.13 updated: feat: dbt add download packages functions and logging functions

[abeizn]

This is an automated email from the ASF dual-hosted git repository.

abeizn pushed a commit to branch release-v0.13
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/release-v0.13 by this push:
     new 1da85758 feat: dbt add download packages functions and logging 
functions
1da85758 is described below

commit 1da85758a5ceed5bd5d81ea6dac3ff5748e72842
Author: abeizn <zikuan...@merico.dev>
AuthorDate: Thu Sep 1 15:01:30 2022 +0800

    feat: dbt add download packages functions and logging functions
---
 Dockerfile                     |  2 +-
 plugins/dbt/tasks/convertor.go | 23 ++++++++++++++++++-----
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 81250fbe..b97ee46d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -37,7 +37,7 @@ ENV GOBIN=/app/bin
 
 RUN make clean && make all
 
-FROM --platform=linux/amd64 mericodev/alpine-dbt-mysql:0.0.1
+FROM --platform=linux/amd64 mericodev/alpine-dbt:0.0.1
 
 EXPOSE 8080
 
diff --git a/plugins/dbt/tasks/convertor.go b/plugins/dbt/tasks/convertor.go
index 2ada7c9b..c65291ef 100644
--- a/plugins/dbt/tasks/convertor.go
+++ b/plugins/dbt/tasks/convertor.go
@@ -33,6 +33,7 @@ import (
 )
 
 func DbtConverter(taskCtx core.SubTaskContext) error {
+       log := taskCtx.GetLogger()
        taskCtx.SetProgress(0, -1)
        data := taskCtx.GetData().(*DbtTaskData)
        models := data.Options.SelectedModels
@@ -101,6 +102,15 @@ func DbtConverter(taskCtx core.SubTaskContext) error {
                return err
        }
 
+       _, err = os.Stat("packages.yml")
+       if err == nil {
+               cmd := exec.Command("dbt", "deps")
+               err = cmd.Start()
+               if err != nil {
+                       return err
+               }
+       }
+
        dbtExecParams := []string{"dbt", "run", "--profiles-dir", projectPath}
        if projectVars != nil {
                jsonProjectVars, err := json.Marshal(projectVars)
@@ -113,18 +123,16 @@ func DbtConverter(taskCtx core.SubTaskContext) error {
        dbtExecParams = append(dbtExecParams, "--select")
        dbtExecParams = append(dbtExecParams, models...)
        cmd := exec.Command(dbtExecParams[0], (dbtExecParams[1:])...)
-       stdout, err := cmd.StdoutPipe()
-       if err != nil {
-               return err
-       }
+       log.Info("dbt run script: ", cmd)
+       stdout, _ := cmd.StdoutPipe()
        err = cmd.Start()
        if err != nil {
                return err
        }
-
        scanner := bufio.NewScanner(stdout)
        for scanner.Scan() {
                line := scanner.Text()
+               log.Info(line)
                if strings.Contains(line, "of") && strings.Contains(line, "OK") 
{
                        taskCtx.IncProgress(1)
                }
@@ -133,6 +141,11 @@ func DbtConverter(taskCtx core.SubTaskContext) error {
                return err
        }
 
+       cmd.Wait()
+       if !cmd.ProcessState.Success() {
+               log.Error("dbt run task error, please check!!!")
+       }
+
        return nil
 }