This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 92e9299 KYLIN-3795 minor code refactor
92e9299 is described below
commit 92e9299b77bff3d924d8cafa73970544b97c6d00
Author: shaofengshi <[email protected]>
AuthorDate: Wed Mar 13 09:49:43 2019 +0800
KYLIN-3795 minor code refactor
---
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 8 ++++----
.../apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java | 4 ++--
.../org/apache/kylin/engine/spark/SparkBatchMergeJobBuilder2.java | 4 ++--
.../org/apache/kylin/engine/spark/SparkExecutableFactory.java | 5 +++--
.../src/main/java/org/apache/kylin/source/hive/HiveInputBase.java | 4 ++--
.../org/apache/kylin/storage/hbase/steps/HBaseSparkSteps.java | 2 +-
6 files changed, 14 insertions(+), 13 deletions(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 81979dc..0a2b9ae 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1377,12 +1377,12 @@ public abstract class KylinConfigBase implements
Serializable {
// ENGINE.LIVY
//
============================================================================
- public boolean enableLivy() {
- return getOptional("kylin.engine.livy-conf.livy.enable",
"false").equalsIgnoreCase("true") ? true : false;
+ public boolean isLivyEnabled() {
+ return
Boolean.parseBoolean(getOptional("kylin.engine.livy-conf.livy-enabled", FALSE));
}
public String getLivyUrl() {
- return getOptional("kylin.engine.livy-conf.livy.url");
+ return getOptional("kylin.engine.livy-conf.livy-url");
}
public Map<String, String> getLivyKey() {
@@ -1396,7 +1396,7 @@ public abstract class KylinConfigBase implements
Serializable {
public Map<String, String> getLivyMap() {
return getPropertiesByPrefix("kylin.engine.livy-conf.livy-map.");
}
-
+
//
============================================================================
// QUERY
//
============================================================================
diff --git
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index 426a73f..c668d69 100644
---
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -88,7 +88,7 @@ public class SparkBatchCubingJobBuilder2 extends
JobBuilderSupport {
}
public SparkExecutable createFactDistinctColumnsSparkStep(String jobId) {
- final SparkExecutable sparkExecutable =
SparkExecutableFactory.instance(seg.getConfig().enableLivy());
+ final SparkExecutable sparkExecutable =
SparkExecutableFactory.instance(seg.getConfig());
final IJoinedFlatTableDesc flatTableDesc =
EngineFactory.getJoinedFlatTableDesc(seg);
final String tablePath = JoinedFlatTable.getTableDir(flatTableDesc,
getJobWorkingDir(jobId));
@@ -115,7 +115,7 @@ public class SparkBatchCubingJobBuilder2 extends
JobBuilderSupport {
}
protected void addLayerCubingSteps(final CubingJob result, final String
jobId, final String cuboidRootPath) {
- final SparkExecutable sparkExecutable =
SparkExecutableFactory.instance(seg.getConfig().enableLivy());
+ final SparkExecutable sparkExecutable =
SparkExecutableFactory.instance(seg.getConfig());
sparkExecutable.setClassName(SparkCubingByLayer.class.getName());
configureSparkJob(seg, sparkExecutable, jobId, cuboidRootPath);
result.addTask(sparkExecutable);
diff --git
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchMergeJobBuilder2.java
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchMergeJobBuilder2.java
index 21599ff..01ea352 100644
---
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchMergeJobBuilder2.java
+++
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchMergeJobBuilder2.java
@@ -77,7 +77,7 @@ public class SparkBatchMergeJobBuilder2 extends
JobBuilderSupport {
}
public SparkExecutable createMergeDictionaryStep(CubeSegment seg, String
jobID, List<String> mergingSegmentIds) {
- final SparkExecutable sparkExecutable =
SparkExecutableFactory.instance(seg.getConfig().enableLivy());
+ final SparkExecutable sparkExecutable =
SparkExecutableFactory.instance(seg.getConfig());
sparkExecutable.setClassName(SparkMergingDictionary.class.getName());
sparkExecutable.setParam(SparkMergingDictionary.OPTION_CUBE_NAME.getOpt(),
seg.getRealization().getName());
@@ -108,7 +108,7 @@ public class SparkBatchMergeJobBuilder2 extends
JobBuilderSupport {
String formattedPath = StringUtil.join(mergingCuboidPaths, ",");
String outputPath = getCuboidRootPath(jobID);
- final SparkExecutable sparkExecutable =
SparkExecutableFactory.instance(seg.getConfig().enableLivy());
+ final SparkExecutable sparkExecutable =
SparkExecutableFactory.instance(seg.getConfig());
sparkExecutable.setClassName(SparkCubingMerge.class.getName());
sparkExecutable.setParam(SparkCubingMerge.OPTION_CUBE_NAME.getOpt(),
seg.getRealization().getName());
sparkExecutable.setParam(SparkCubingMerge.OPTION_SEGMENT_ID.getOpt(),
seg.getUuid());
diff --git
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutableFactory.java
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutableFactory.java
index bc59c12..8c34454 100644
---
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutableFactory.java
+++
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutableFactory.java
@@ -17,6 +17,7 @@
*/
package org.apache.kylin.engine.spark;
+import org.apache.kylin.common.KylinConfig;
import org.slf4j.LoggerFactory;
/**
@@ -25,8 +26,8 @@ public class SparkExecutableFactory {
private static final org.slf4j.Logger logger =
LoggerFactory.getLogger(SparkExecutableFactory.class);
- public static SparkExecutable instance(boolean livy) {
- return livy ? new SparkExecutableLivy() : new SparkExecutable();
+ public static SparkExecutable instance(KylinConfig kylinConfig) {
+ return kylinConfig.isLivyEnabled() ? new SparkExecutableLivy() : new
SparkExecutable();
}
}
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveInputBase.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveInputBase.java
index 4782920..bf674f5 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveInputBase.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveInputBase.java
@@ -87,7 +87,7 @@ public class HiveInputBase {
if (cubeConfig.isHiveRedistributeEnabled()) {
final KylinConfig kylinConfig =
KylinConfig.getInstanceFromEnv();
//jobFlow.addTask(createRedistributeFlatHiveTableStep(hiveInitStatements,
cubeName, flatDesc, cubeInstance.getDescriptor()));
- if (kylinConfig.enableLivy() && cubeInstance.getEngineType()
== IEngineAware.ID_SPARK) {
+ if (kylinConfig.isLivyEnabled() &&
cubeInstance.getEngineType() == IEngineAware.ID_SPARK) {
jobFlow.addTask(createRedistributeFlatHiveTableByLivyStep(hiveInitStatements,
cubeName, flatDesc, cubeInstance.getDescriptor()));
} else {
jobFlow.addTask(createRedistributeFlatHiveTableStep(hiveInitStatements,
cubeName, flatDesc, cubeInstance.getDescriptor()));
@@ -106,7 +106,7 @@ public class HiveInputBase {
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
CubeInstance cubeInstance =
CubeManager.getInstance(kylinConfig).getCube(cubeName);
- if (kylinConfig.enableLivy() && cubeInstance.getEngineType() ==
IEngineAware.ID_SPARK) {
+ if (kylinConfig.isLivyEnabled() && cubeInstance.getEngineType() ==
IEngineAware.ID_SPARK) {
jobFlow.addTask(createFlatHiveTableByLivyStep(hiveInitStatements,
jobWorkingDir, cubeName, flatDesc));
} else {
jobFlow.addTask(createFlatHiveTableStep(hiveInitStatements,
jobWorkingDir, cubeName, flatDesc));
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseSparkSteps.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseSparkSteps.java
index 1f35de4..1d480c3 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseSparkSteps.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseSparkSteps.java
@@ -40,7 +40,7 @@ public class HBaseSparkSteps extends HBaseJobSteps {
String inputPath = cuboidRootPath + (cuboidRootPath.endsWith("/") ? ""
: "/");
SparkBatchCubingJobBuilder2 jobBuilder2 = new
SparkBatchCubingJobBuilder2(seg, null);
- final SparkExecutable sparkExecutable =
SparkExecutableFactory.instance(seg.getConfig().enableLivy());
+ final SparkExecutable sparkExecutable =
SparkExecutableFactory.instance(seg.getConfig());
sparkExecutable.setClassName(SparkCubeHFile.class.getName());
sparkExecutable.setParam(SparkCubeHFile.OPTION_CUBE_NAME.getOpt(),
seg.getRealization().getName());
sparkExecutable.setParam(SparkCubeHFile.OPTION_SEGMENT_ID.getOpt(),
seg.getUuid());
