This is an automated email from the ASF dual-hosted git repository.
yangzhg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new cfeb515 [Enhancement] add spark load config
`spark_load_checker_interval_second` (#6809)
cfeb515 is described below
commit cfeb515f5ee7cebf19e0081f8b60943b0b66911c
Author: Xiang Wei <[email protected]>
AuthorDate: Tue Oct 12 22:39:51 2021 -0500
[Enhancement] add spark load config `spark_load_checker_interval_second`
(#6809)
the internal seconds for spark load to check and update etl job status, 60
seconds for default
Co-authored-by: weixiang <[email protected]>
---
fe/fe-core/src/main/java/org/apache/doris/common/Config.java | 6 ++++++
.../src/main/java/org/apache/doris/load/loadv2/LoadEtlChecker.java | 2 +-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
index d319d3c..f67bee9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
@@ -493,6 +493,12 @@ public class Config extends ConfigBase {
@ConfField public static int load_checker_interval_second = 5;
/**
+ * The spark load scheduler running interval.
+ * Default 60 seconds, because spark load job is heavy and yarn client
returns slowly.
+ */
+ @ConfField public static int spark_load_checker_interval_second = 60;
+
+ /**
* Concurrency of HIGH priority pending load jobs.
* Load job priority is defined as HIGH or NORMAL.
* All mini batch load jobs are HIGH priority, other types of load jobs
are NORMAL priority.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadEtlChecker.java
b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadEtlChecker.java
index c72896c..2097365 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadEtlChecker.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadEtlChecker.java
@@ -32,7 +32,7 @@ public class LoadEtlChecker extends MasterDaemon {
private LoadManager loadManager;
public LoadEtlChecker(LoadManager loadManager) {
- super("Load etl checker", Config.load_checker_interval_second * 1000);
+ super("Load etl checker", Config.spark_load_checker_interval_second *
1000);
this.loadManager = loadManager;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
