This is an automated email from the ASF dual-hosted git repository. shaofengshi pushed a commit to branch KYLIN-3368 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 11a7cd506887af3d130d4470c7eee5bfca4aefd9 Author: shaofengshi <[email protected]> AuthorDate: Fri May 11 21:29:40 2018 +0800 KYLIN-3368 Move Spark cubing metadata dump to job folder --- .../java/org/apache/kylin/engine/mr/JobBuilderSupport.java | 5 +++++ .../kylin/engine/spark/SparkBatchCubingJobBuilder2.java | 12 ++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java index 8228f87..8a420df 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java @@ -261,4 +261,9 @@ public class JobBuilderSupport { public static String getInMemCuboidPath(String cuboidRootPath) { return cuboidRootPath + PathNameCuboidInMem; } + + public String getDumpMetadataPath(String jobId) { + return getRealizationRootPath(jobId) + "/metadata"; + } + } diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java index 7d76ce4..57d4fb0 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java @@ -67,7 +67,7 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 { return ""; } - public static void configureSparkJob(final CubeSegment seg, final SparkExecutable sparkExecutable, + public void configureSparkJob(final CubeSegment seg, final SparkExecutable sparkExecutable, final String jobId, final String cuboidRootPath) { IJoinedFlatTableDesc flatTableDesc = EngineFactory.getJoinedFlatTableDesc(seg); sparkExecutable.setParam(SparkCubingByLayer.OPTION_CUBE_NAME.getOpt(), seg.getRealization().getName()); @@ -75,15 +75,12 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 { sparkExecutable.setParam(SparkCubingByLayer.OPTION_INPUT_TABLE.getOpt(), seg.getConfig().getHiveDatabaseForIntermediateTable() + "." + flatTableDesc.getTableName()); sparkExecutable.setParam(SparkCubingByLayer.OPTION_META_URL.getOpt(), - getSegmentMetadataUrl(seg.getConfig(), seg.getUuid())); + getSegmentMetadataUrl(seg.getConfig(), jobId)); sparkExecutable.setParam(SparkCubingByLayer.OPTION_OUTPUT_PATH.getOpt(), cuboidRootPath); sparkExecutable.setJobId(jobId); StringBuilder jars = new StringBuilder(); - StringUtil.appendWithSeparator(jars, findJar("org.htrace.HTraceConfiguration", null)); // htrace-core.jar - StringUtil.appendWithSeparator(jars, findJar("org.apache.htrace.Trace", null)); // htrace-core.jar - StringUtil.appendWithSeparator(jars, findJar("org.cloudera.htrace.HTraceConfiguration", null)); // htrace-core.jar StringUtil.appendWithSeparator(jars, findJar("com.yammer.metrics.core.Gauge", null)); // metrics-core.jar StringUtil.appendWithSeparator(jars, findJar("com.google.common.collect.Maps", "guava")); //guava.jar @@ -92,10 +89,9 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 { sparkExecutable.setName(ExecutableConstants.STEP_NAME_BUILD_SPARK_CUBE); } - private static String getSegmentMetadataUrl(KylinConfig kylinConfig, String segmentID) { + private String getSegmentMetadataUrl(KylinConfig kylinConfig, String jobId) { Map<String, String> param = new HashMap<>(); - param.put("path", kylinConfig.getHdfsWorkingDirectory() + "metadata/" + segmentID); + param.put("path", getDumpMetadataPath(jobId)); return new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "hdfs", param).toString(); -// return kylinConfig.getHdfsWorkingDirectory() + "metadata/" + segmentID + "@hdfs"; } } -- To stop receiving notification emails like this one, please contact [email protected].
