Repository: hive Updated Branches: refs/heads/hive-14535 38cd0a67c -> 36ad3a405
HIVE-14990 : run all tests for MM tables and fix the issues that are found (Sergey Shelukhin) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/36ad3a40 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/36ad3a40 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/36ad3a40 Branch: refs/heads/hive-14535 Commit: 36ad3a405daceb2ce0c1f7979778be03a4ff9bbb Parents: 38cd0a6 Author: Sergey Shelukhin <[email protected]> Authored: Tue Oct 25 19:26:36 2016 -0700 Committer: Sergey Shelukhin <[email protected]> Committed: Tue Oct 25 19:26:36 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java | 6 ++++++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java | 1 + .../org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java | 7 +++++-- 3 files changed, 12 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/36ad3a40/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index c2ce259..28fcfa8 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -1894,4 +1894,10 @@ public class MetaStoreUtils { String value = params.get(hive_metastoreConstants.TABLE_IS_MM); return value != null && value.equalsIgnoreCase("true"); } + + public static boolean isMmTable(Properties params) { + // TODO: perhaps it should be a 3rd value for 'transactional'? + String value = params.getProperty(hive_metastoreConstants.TABLE_IS_MM); + return value != null && value.equalsIgnoreCase("true"); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/36ad3a40/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index e0af81e..6774d4d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -3961,6 +3961,7 @@ public final class Utilities { } Utilities.LOG14535.info("Looking for manifests in: " + manifestDir + " (" + mmWriteId + ")"); + // TODO# may be wrong if there are no splits (empty insert/CTAS) FileStatus[] manifestFiles = fs.listStatus(manifestDir); List<Path> manifests = new ArrayList<>(); if (manifestFiles != null) { http://git-wip-us.apache.org/repos/asf/hive/blob/36ad3a40/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java index e91064b..59d6142 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java @@ -41,6 +41,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.log.PerfLogger; @@ -104,8 +105,10 @@ public class CombineHiveInputFormat<K extends WritableComparable, V extends Writ Class<? extends InputFormat> inputFormatClass = part.getInputFileFormatClass(); InputFormat<WritableComparable, Writable> inputFormat = getInputFormatFromCache(inputFormatClass, conf); - if (inputFormat instanceof AvoidSplitCombination && - ((AvoidSplitCombination) inputFormat).shouldSkipCombine(paths[i + start], conf)) { + boolean isAvoidSplitCombine = inputFormat instanceof AvoidSplitCombination && + ((AvoidSplitCombination) inputFormat).shouldSkipCombine(paths[i + start], conf); + boolean isMmTable = MetaStoreUtils.isMmTable(part.getTableDesc().getProperties()); + if (isAvoidSplitCombine || isMmTable) { if (LOG.isDebugEnabled()) { LOG.debug("The path [" + paths[i + start] + "] is being parked for HiveInputFormat.getSplits");
