shaofengshi closed pull request #347: KYLIN-3666 HDFS metadata url not be
recognized
URL: https://github.com/apache/kylin/pull/347
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 6a9158d609..0624f94dc7 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -490,7 +490,14 @@ public static KylinConfig loadKylinPropsAndMetadata()
throws IOException {
public static KylinConfig
loadKylinConfigFromHdfs(SerializableConfiguration conf, String uri) {
HadoopUtil.setCurrentConfiguration(conf.get());
- return loadKylinConfigFromHdfs(uri);
+ KylinConfig config = loadKylinConfigFromHdfs(uri);
+
+ // This is a bad example where the thread local KylinConfig cannot be
auto-closed due to
+ // limitation of MR API. It works because MR task runs its own
process. Do not copy.
+ @SuppressWarnings("unused")
+ SetAndUnsetThreadLocalConfig shouldAutoClose =
KylinConfig.setAndUnsetThreadLocalConfig(config);
+
+ return config;
}
public static KylinConfig loadKylinConfigFromHdfs(String uri) {
@@ -517,11 +524,7 @@ public static KylinConfig loadKylinConfigFromHdfs(String
uri) {
} catch (IOException e) {
throw new RuntimeException(e);
}
-
- // This is a bad example where the thread local KylinConfig cannot be
auto-closed due to
- // limitation of MR API. It works because MR task runs its own
process. Do not copy.
- @SuppressWarnings("unused")
- SetAndUnsetThreadLocalConfig shouldAutoClose =
KylinConfig.setAndUnsetThreadLocalConfig(config);
+
kylinConfigCache.put(uri, config);
return config;
}
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java
index bcdb29f52c..28c567a7d0 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java
@@ -43,6 +43,7 @@
import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -67,7 +68,7 @@ protected ExecuteResult doWork(ExecutableContext context)
throws ExecuteExceptio
final String metadataUrl =
this.getParams().get(BatchConstants.ARG_META_URL);
final KylinConfig kylinConfHbase = cube.getConfig();
- final KylinConfig kylinConfHdfs =
KylinConfig.createInstanceFromUri(metadataUrl);
+ final KylinConfig kylinConfHdfs =
AbstractHadoopJob.loadKylinConfigFromHdfs(metadataUrl);
Collections.sort(mergingSegments);
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
