code review: change parameter name from isPartial to isFullTable
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/09377224 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/09377224 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/09377224 Branch: refs/heads/master-hbase0.98 Commit: 09377224ce2e26fdd1d00b952d8f5bca7654bd07 Parents: 545d145 Author: shaofengshi <shaofeng...@apache.org> Authored: Wed Feb 15 13:36:17 2017 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Wed Feb 15 13:36:17 2017 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/engine/mr/IMRInput.java | 3 ++- .../main/java/org/apache/kylin/engine/mr/MRUtil.java | 14 ++++++++------ .../org/apache/kylin/source/hive/HiveMRInput.java | 10 +++++----- .../hive/cardinality/ColumnCardinalityMapper.java | 2 +- .../hive/cardinality/HiveColumnCardinalityJob.java | 2 +- .../org/apache/kylin/source/kafka/KafkaMRInput.java | 10 +++++----- 6 files changed, 22 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/09377224/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMRInput.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMRInput.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMRInput.java index f6578f9..10d4879 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMRInput.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMRInput.java @@ -35,7 +35,8 @@ public interface IMRInput { /** Return an InputFormat that reads from specified table. */ public IMRTableInputFormat getTableInputFormat(TableDesc table); - public IMRTableInputFormat getTableInputFormat(TableDesc table, boolean isPartial); + /** Return an InputFormat that reads from specified table; "isFullTable" indicates whether it is a full table read or not. */ + public IMRTableInputFormat getTableInputFormat(TableDesc table, boolean isFullTable); /** Return a helper to participate in batch cubing merge job flow. */ public IMRBatchMergeInputSide getBatchMergeInputSide(ISegment seg); http://git-wip-us.apache.org/repos/asf/kylin/blob/09377224/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java index eb7bc4b..5ba9424 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java @@ -43,20 +43,22 @@ public class MRUtil { return SourceFactory.createEngineAdapter(seg, IMRInput.class).getBatchCubingInputSide(flatDesc); } + @Deprecated public static IMRTableInputFormat getTableInputFormat(String tableName) { - return getTableInputFormat(getTableDesc(tableName)); + return getTableInputFormat(tableName, true); } - public static IMRTableInputFormat getTableInputFormat(String tableName, boolean isPartial) { - return getTableInputFormat(getTableDesc(tableName), isPartial); + public static IMRTableInputFormat getTableInputFormat(String tableName, boolean isFullTable) { + return getTableInputFormat(getTableDesc(tableName), isFullTable); } + @Deprecated public static IMRTableInputFormat getTableInputFormat(TableDesc tableDesc) { - return SourceFactory.createEngineAdapter(tableDesc, IMRInput.class).getTableInputFormat(tableDesc); + return getTableInputFormat(tableDesc, true); } - public static IMRTableInputFormat getTableInputFormat(TableDesc tableDesc, boolean isPartial) { - return SourceFactory.createEngineAdapter(tableDesc, IMRInput.class).getTableInputFormat(tableDesc, isPartial); + public static IMRTableInputFormat getTableInputFormat(TableDesc tableDesc, boolean isFullTable) { + return SourceFactory.createEngineAdapter(tableDesc, IMRInput.class).getTableInputFormat(tableDesc, isFullTable); } private static TableDesc getTableDesc(String tableName) { http://git-wip-us.apache.org/repos/asf/kylin/blob/09377224/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java index c1ea45a..39a0273 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java @@ -60,8 +60,8 @@ import com.google.common.collect.Sets; public class HiveMRInput implements IMRInput { - public static String getTableNameForHCat(TableDesc table, boolean isPartial) { - String tableName = (table.isView() || isPartial) ? table.getMaterializedName() : table.getName(); + public static String getTableNameForHCat(TableDesc table, boolean isFullTable) { + String tableName = (table.isView() || isFullTable == false) ? table.getMaterializedName() : table.getName(); return String.format("%s.%s", table.getDatabase(), tableName).toUpperCase(); } @@ -72,12 +72,12 @@ public class HiveMRInput implements IMRInput { @Override public IMRTableInputFormat getTableInputFormat(TableDesc table) { - return new HiveTableInputFormat(getTableNameForHCat(table, false)); + return new HiveTableInputFormat(getTableNameForHCat(table, true)); } @Override - public IMRTableInputFormat getTableInputFormat(TableDesc table, boolean isPartial) { - return new HiveTableInputFormat(getTableNameForHCat(table, isPartial)); + public IMRTableInputFormat getTableInputFormat(TableDesc table, boolean isFullTable) { + return new HiveTableInputFormat(getTableNameForHCat(table, isFullTable)); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/09377224/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java index 145ed31..c712605 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java @@ -62,7 +62,7 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T, Object, IntWritab String tableName = conf.get(BatchConstants.CFG_TABLE_NAME); tableDesc = MetadataManager.getInstance(config).getTableDesc(tableName); - tableInputFormat = MRUtil.getTableInputFormat(tableDesc); + tableInputFormat = MRUtil.getTableInputFormat(tableDesc, true); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/09377224/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java index f439ccb..a837681 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java @@ -87,7 +87,7 @@ public class HiveColumnCardinalityJob extends AbstractHadoopJob { job.getConfiguration().set("mapreduce.output.fileoutputformat.compress", "false"); // Mapper - IMRTableInputFormat tableInputFormat = MRUtil.getTableInputFormat(table); + IMRTableInputFormat tableInputFormat = MRUtil.getTableInputFormat(table, true); tableInputFormat.configureJob(job); job.setMapperClass(ColumnCardinalityMapper.class); http://git-wip-us.apache.org/repos/asf/kylin/blob/09377224/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java ---------------------------------------------------------------------- diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java index 2d5a301..c7b327f 100644 --- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java +++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java @@ -74,6 +74,11 @@ public class KafkaMRInput implements IMRInput { @Override public IMRTableInputFormat getTableInputFormat(TableDesc table) { + return getTableInputFormat(table, true); + } + + @Override + public IMRTableInputFormat getTableInputFormat(TableDesc table, boolean isFullTable) { KafkaConfigManager kafkaConfigManager = KafkaConfigManager.getInstance(KylinConfig.getInstanceFromEnv()); KafkaConfig kafkaConfig = kafkaConfigManager.getKafkaConfig(table.getIdentity()); List<TblColRef> columns = Lists.transform(Arrays.asList(table.getColumns()), new Function<ColumnDesc, TblColRef>() { @@ -88,11 +93,6 @@ public class KafkaMRInput implements IMRInput { } @Override - public IMRTableInputFormat getTableInputFormat(TableDesc table, boolean isPartial) { - return getTableInputFormat(table); - } - - @Override public IMRBatchMergeInputSide getBatchMergeInputSide(ISegment seg) { return new KafkaMRBatchMergeInputSide((CubeSegment) seg); }