Repository: kylin Updated Branches: refs/heads/master e4b1d7e0b -> b02e7bd24
KYLIN-3032 Cube size shows 0 but actually it is not empty Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b02e7bd2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b02e7bd2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b02e7bd2 Branch: refs/heads/master Commit: b02e7bd24cc724e03023706eac6389bfe71ebb3e Parents: 2b9470c Author: shaofengshi <[email protected]> Authored: Wed Nov 22 13:57:25 2017 +0800 Committer: shaofengshi <[email protected]> Committed: Wed Nov 22 13:58:11 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/engine/mr/common/HadoopCmdOutput.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/b02e7bd2/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java index 11d6d2c..d82b988 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java @@ -22,11 +22,12 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; -import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.FileSystemCounter; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.TaskCounter; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.engine.mr.steps.FactDistinctColumnsMapper.RawDataCounter; import org.apache.kylin.job.constant.ExecutableConstants; import org.slf4j.Logger; @@ -99,9 +100,13 @@ public class HadoopCmdOutput { mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue()); rawInputBytesRead = String.valueOf(counters.findCounter(RawDataCounter.BYTES).getValue()); - String fsScheme = FileSystem.get(job.getConfiguration()).getScheme(); + String outputFolder = job.getConfiguration().get("mapreduce.output.fileoutputformat.outputdir", KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()); + logger.debug("outputFolder is " + outputFolder); + Path outputPath = new Path(outputFolder); + String fsScheme = outputPath.getFileSystem(job.getConfiguration()).getScheme(); long bytesWritten = counters.findCounter(fsScheme, FileSystemCounter.BYTES_WRITTEN).getValue(); if (bytesWritten == 0) { + logger.debug("Seems no counter found for " + fsScheme); bytesWritten = counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue(); } hdfsBytesWritten = String.valueOf(bytesWritten);
