Repository: hive Updated Branches: refs/heads/master 00145ee85 -> 20a34d329
HIVE-18237 : missing results for insert_only table after DP insert (Steve Yeom, reviewed by Sergey Shelukhin) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/20a34d32 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/20a34d32 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/20a34d32 Branch: refs/heads/master Commit: 20a34d329d41c6b46eb576761fa55ae327d5208a Parents: 00145ee Author: sergey <[email protected]> Authored: Tue Jan 30 15:41:38 2018 -0800 Committer: sergey <[email protected]> Committed: Tue Jan 30 15:41:38 2018 -0800 ---------------------------------------------------------------------- ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java | 7 +++++-- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java | 2 +- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java | 9 +++++---- .../org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java | 6 ++++++ .../java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java | 9 +++++++++ 5 files changed, 26 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/20a34d32/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index 0df30f1..114d455 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -503,8 +503,11 @@ public class MoveTask extends Task<MoveWork> implements Serializable { (tbd.getLbCtx() == null) ? 0 : tbd.getLbCtx().calculateListBucketingLevel(), work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID && !tbd.isMmTable(), - work.getLoadTableWork().getTxnId(), tbd.getStmtId(), hasFollowingStatsTask(), - work.getLoadTableWork().getWriteType()); + work.getLoadTableWork().getTxnId(), + tbd.getStmtId(), + hasFollowingStatsTask(), + work.getLoadTableWork().getWriteType(), + tbd.isInsertOverwrite()); // publish DP columns to its subscribers if (dps != null && dps.size() > 0) { http://git-wip-us.apache.org/repos/asf/hive/blob/20a34d32/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index d7b3e4b..7348faa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -4075,7 +4075,7 @@ public final class Utilities { throws IOException { int skipLevels = dpLevels + lbLevels; if (filter == null) { - filter = new JavaUtils.IdPathFilter(txnId, stmtId, true); + filter = new JavaUtils.IdPathFilter(txnId, stmtId, true, false, isBaseDir); } if (skipLevels == 0) { return statusToPath(fs.listStatus(path, filter)); http://git-wip-us.apache.org/repos/asf/hive/blob/20a34d32/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 23983d8..63bcedc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -2178,7 +2178,8 @@ private void constructOneLBLocationMap(FileStatus fSta, * @throws HiveException */ private Set<Path> getValidPartitionsInPath( - int numDP, int numLB, Path loadPath, Long txnId, int stmtId, boolean isMmTable) throws HiveException { + int numDP, int numLB, Path loadPath, Long txnId, int stmtId, + boolean isMmTable, boolean isInsertOverwrite) throws HiveException { Set<Path> validPartitions = new HashSet<Path>(); try { FileSystem fs = loadPath.getFileSystem(conf); @@ -2199,7 +2200,7 @@ private void constructOneLBLocationMap(FileStatus fSta, // where this is used; we always want to load everything; also the only case where // we have multiple statements anyway is union. Path[] leafStatus = Utilities.getMmDirectoryCandidates( - fs, loadPath, numDP, numLB, null, txnId, -1, conf, false); + fs, loadPath, numDP, numLB, null, txnId, -1, conf, isInsertOverwrite); for (Path p : leafStatus) { Path dpPath = p.getParent(); // Skip the MM directory that we have found. for (int i = 0; i < numLB; ++i) { @@ -2247,7 +2248,7 @@ private void constructOneLBLocationMap(FileStatus fSta, public Map<Map<String, String>, Partition> loadDynamicPartitions(final Path loadPath, final String tableName, final Map<String, String> partSpec, final LoadFileType loadFileType, final int numDP, final int numLB, final boolean isAcid, final long txnId, final int stmtId, - final boolean hasFollowingStatsTask, final AcidUtils.Operation operation) + final boolean hasFollowingStatsTask, final AcidUtils.Operation operation, boolean isInsertOverwrite) throws HiveException { final Map<Map<String, String>, Partition> partitionsMap = @@ -2263,7 +2264,7 @@ private void constructOneLBLocationMap(FileStatus fSta, // Get all valid partition paths and existing partitions for them (if any) final Table tbl = getTable(tableName); final Set<Path> validPartitions = getValidPartitionsInPath(numDP, numLB, loadPath, txnId, stmtId, - AcidUtils.isInsertOnlyTable(tbl.getParameters())); + AcidUtils.isInsertOnlyTable(tbl.getParameters()), isInsertOverwrite); final int partsToLoad = validPartitions.size(); final AtomicInteger partitionsLoaded = new AtomicInteger(0); http://git-wip-us.apache.org/repos/asf/hive/blob/20a34d32/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 69d4fa5..dbf9363 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -6995,6 +6995,12 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { tableDesc.setWriter(fileSinkDesc); } + if (fileSinkDesc.getInsertOverwrite()) { + if (ltd != null) { + ltd.setInsertOverwrite(true); + } + } + if (SessionState.get().isHiveServerQuery() && null != table_desc && table_desc.getSerdeClassName().equalsIgnoreCase(ThriftJDBCBinarySerDe.class.getName()) && http://git-wip-us.apache.org/repos/asf/hive/blob/20a34d32/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java index 1a9c11e..a40c486 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java @@ -40,6 +40,7 @@ public class LoadTableDesc extends LoadDesc implements Serializable { //table specs are to be used private int stmtId; private Long currentTransactionId; + private boolean isInsertOverwrite; // TODO: the below seem like they should just be combined into partitionDesc private org.apache.hadoop.hive.ql.plan.TableDesc table; @@ -209,6 +210,14 @@ public class LoadTableDesc extends LoadDesc implements Serializable { this.inheritTableSpecs = inheritTableSpecs; } + public boolean isInsertOverwrite() { + return this.isInsertOverwrite; + } + + public void setInsertOverwrite(boolean v) { + this.isInsertOverwrite = v; + } + /** * @return the lbCtx */
