shaofengshi closed pull request #192: KYLIN-3471 Minor, remove duplicate code
URL: https://github.com/apache/kylin/pull/192
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 180b56a485..329dd569d3 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -27,7 +27,6 @@
import static
org.apache.kylin.engine.mr.common.JobRelatedMetaUtil.collectCubeMetadata;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
@@ -63,7 +62,6 @@
import org.apache.kylin.common.KylinConfig.SetAndUnsetThreadLocalConfig;
import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.StorageURL;
-import org.apache.kylin.common.persistence.ResourceTool;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.OptionsHelper;
@@ -561,32 +559,7 @@ protected void
attachSegmentsMetadataWithDict(List<CubeSegment> segments, String
dumpList.addAll(segment.getDictionaryPaths());
dumpList.add(segment.getStatisticsResourcePath());
}
- dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt)
segments.get(0).getConfig(), metaUrl);
- }
-
- private void dumpAndUploadKylinPropsAndMetadata(Set<String> dumpList,
KylinConfigExt kylinConfig, String metadataUrl)
- throws IOException {
- File tmp = File.createTempFile("kylin_job_meta", "");
- FileUtils.forceDelete(tmp); // we need a directory, so delete the file
first
-
- File metaDir = new File(tmp, "meta");
- metaDir.mkdirs();
-
- // dump metadata
- JobRelatedMetaUtil.dumpResources(kylinConfig, metaDir, dumpList);
-
- // write kylin.properties
- Properties props = kylinConfig.exportToProperties();
- props.setProperty("kylin.metadata.url", metadataUrl);
-
- File kylinPropsFile = new File(metaDir, "kylin.properties");
- try (FileOutputStream os = new FileOutputStream(kylinPropsFile)) {
- props.store(os, kylinPropsFile.getAbsolutePath());
- }
-
- KylinConfig dstConfig = KylinConfig.createKylinConfig(props);
- //upload metadata
-
ResourceTool.copy(KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()),
dstConfig);
+ JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList,
(KylinConfigExt) segments.get(0).getConfig(), metaUrl);
}
protected void attachSegmentMetadataWithDict(CubeSegment segment,
Configuration conf) throws IOException {
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobRelatedMetaUtil.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobRelatedMetaUtil.java
index 2cd1841611..64469a0165 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobRelatedMetaUtil.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobRelatedMetaUtil.java
@@ -18,9 +18,12 @@
package org.apache.kylin.engine.mr.common;
+import org.apache.commons.io.FileUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.ResourceTool;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableRef;
@@ -29,8 +32,10 @@
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.util.LinkedHashSet;
+import java.util.Properties;
import java.util.Set;
public class JobRelatedMetaUtil {
@@ -69,4 +74,28 @@ public static void dumpResources(KylinConfig kylinConfig,
File metaDir, Set<Stri
logger.debug("Dump resources to {} took {} ms", metaDir,
System.currentTimeMillis() - startTime);
}
+
+ public static void dumpAndUploadKylinPropsAndMetadata(Set<String>
dumpList, KylinConfigExt kylinConfig, String metadataUrl)
+ throws IOException {
+ File tmp = File.createTempFile("kylin_job_meta", "");
+ FileUtils.forceDelete(tmp); // we need a directory, so delete the file
first
+
+ File metaDir = new File(tmp, "meta");
+ metaDir.mkdirs();
+
+ // dump metadata
+ dumpResources(kylinConfig, metaDir, dumpList);
+
+ // write kylin.properties
+ Properties props = kylinConfig.exportToProperties();
+ props.setProperty("kylin.metadata.url", metadataUrl);
+ File kylinPropsFile = new File(metaDir, "kylin.properties");
+ try (FileOutputStream os = new FileOutputStream(kylinPropsFile)) {
+ props.store(os, kylinPropsFile.getAbsolutePath());
+ }
+
+ KylinConfig dstConfig = KylinConfig.createKylinConfig(props);
+ //upload metadata
+
ResourceTool.copy(KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()),
dstConfig);
+ }
}
diff --git
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index d8eba713b3..637382cba9 100644
---
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -17,28 +17,23 @@
*/
package org.apache.kylin.engine.spark;
-import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-import java.util.Properties;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.util.Shell;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.ResourceTool;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
@@ -381,7 +376,7 @@ private void attachSegmentMetadataWithDict(CubeSegment
segment) throws IOExcepti
dumpList.addAll(JobRelatedMetaUtil.collectCubeMetadata(segment.getCubeInstance()));
dumpList.addAll(segment.getDictionaryPaths());
dumpList.add(segment.getStatisticsResourcePath());
- dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt)
segment.getConfig());
+ JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList,
(KylinConfigExt) segment.getConfig(),
this.getParam(SparkCubingByLayer.OPTION_META_URL.getOpt()));
}
private void attachSegmentsMetadataWithDict(List<CubeSegment> segments)
throws IOException {
@@ -395,32 +390,7 @@ private void
attachSegmentsMetadataWithDict(List<CubeSegment> segments) throws I
dumpList.add(segment.getStatisticsResourcePath());
}
}
- dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt)
segments.get(0).getConfig());
- }
-
- private void dumpAndUploadKylinPropsAndMetadata(Set<String> dumpList,
KylinConfigExt kylinConfig)
- throws IOException {
- File tmp = File.createTempFile("kylin_job_meta", "");
- FileUtils.forceDelete(tmp); // we need a directory, so delete the file
first
-
- File metaDir = new File(tmp, "meta");
- metaDir.mkdirs();
-
- // dump metadata
- JobRelatedMetaUtil.dumpResources(kylinConfig, metaDir, dumpList);
-
- // write kylin.properties
- Properties props = kylinConfig.exportToProperties();
- String metadataUrl =
this.getParam(SparkCubingByLayer.OPTION_META_URL.getOpt());
- props.setProperty("kylin.metadata.url", metadataUrl);
- File kylinPropsFile = new File(metaDir, "kylin.properties");
- try (FileOutputStream os = new FileOutputStream(kylinPropsFile)) {
- props.store(os, kylinPropsFile.getAbsolutePath());
- }
-
- KylinConfig dstConfig = KylinConfig.createKylinConfig(props);
- //upload metadata
-
ResourceTool.copy(KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()),
dstConfig);
+ JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList,
(KylinConfigExt) segments.get(0).getConfig(),
this.getParam(SparkCubingByLayer.OPTION_META_URL.getOpt()));
}
private void readCounters(final Map<String, String> info) {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
