[incubator-devlake] branch release-v0.14 updated: feat: check count after starrocks sync (#3550)

[abeizn]

This is an automated email from the ASF dual-hosted git repository.

abeizn pushed a commit to branch release-v0.14
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/release-v0.14 by this push:
     new a61cf40d feat: check count after starrocks sync (#3550)
a61cf40d is described below

commit a61cf40d9f908c6de6fcd30fd970036138c023e3
Author: long2ice <long2...@gmail.com>
AuthorDate: Mon Oct 24 12:42:29 2022 +0800

    feat: check count after starrocks sync (#3550)
---
 plugins/starrocks/tasks.go | 37 ++++++++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/plugins/starrocks/tasks.go b/plugins/starrocks/tasks.go
index f23cc095..6493ed8f 100644
--- a/plugins/starrocks/tasks.go
+++ b/plugins/starrocks/tasks.go
@@ -22,6 +22,12 @@ import (
        "database/sql"
        "encoding/json"
        "fmt"
+       "io"
+       "net/http"
+       "net/url"
+       "regexp"
+       "strings"
+
        "github.com/apache/incubator-devlake/errors"
        "github.com/apache/incubator-devlake/impl/dalgorm"
        "github.com/apache/incubator-devlake/plugins/core"
@@ -30,11 +36,6 @@ import (
        "gorm.io/driver/mysql"
        "gorm.io/driver/postgres"
        "gorm.io/gorm"
-       "io"
-       "net/http"
-       "net/url"
-       "regexp"
-       "strings"
 )
 
 type Table struct {
@@ -334,6 +335,32 @@ func loadData(starrocks *sql.DB, c core.SubTaskContext, 
starrocksTable, starrock
        if err != nil {
                return err
        }
+       // check data count
+       rows, err := db.RawCursor(fmt.Sprintf("select count(*) from %s", table))
+       if err != nil {
+               return err
+       }
+       var sourceCount int
+       for rows.Next() {
+               err = rows.Scan(&sourceCount)
+               if err != nil {
+                       return err
+               }
+       }
+       rows, err = starrocks.Query(fmt.Sprintf("select count(*) from %s", 
starrocksTable))
+       if err != nil {
+               return err
+       }
+       var starrocksCount int
+       for rows.Next() {
+               err = rows.Scan(&starrocksCount)
+               if err != nil {
+                       return err
+               }
+       }
+       if sourceCount != starrocksCount {
+               c.GetLogger().Warn(nil, "source count %d not equal to starrocks 
count %d", sourceCount, starrocksCount)
+       }
        c.GetLogger().Info("load %s to starrocks success", table)
        return nil
 }