This is an automated email from the ASF dual-hosted git repository.
liaoxin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-streamloader.git
The following commit(s) were added to refs/heads/master by this push:
new 5cd581b disable concurrency when the format is not CSV (#15)
5cd581b is described below
commit 5cd581b1409303c6a35989dd6a5a44a9b11412d8
Author: hui lai <[email protected]>
AuthorDate: Tue Jun 11 15:03:14 2024 +0800
disable concurrency when the format is not CSV (#15)
---
main.go | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/main.go b/main.go
index 7a7737c..1677b4f 100644
--- a/main.go
+++ b/main.go
@@ -70,6 +70,7 @@ var (
maxBytesPerTask int
debug bool
workers int
+ enableConcurrency bool
diskThroughput int
streamLoadThroughput int
checkUTF8 bool
@@ -245,6 +246,9 @@ func paramCheck() {
continue
}
kv := strings.Split(v, ":")
+ if strings.ToLower(kv[0]) == "format" &&
strings.ToLower(kv[1]) == "csv" {
+ enableConcurrency = true
+ }
if len(kv) > 2 {
headers[kv[0]] = strings.Join(kv[1:], ":")
} else {
@@ -294,6 +298,11 @@ func calculateAndCheckWorkers(reader *file.FileReader,
size int64) {
return
}
+ if !enableConcurrency {
+ loadInfo.Workers = 1
+ return
+ }
+
ratio := float64(size) / float64(maxBytesPerTask)
tmpWorkers := 0
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
