Repository: hive Updated Branches: refs/heads/master b7723e498 -> 211116543
HIVE-20541: REPL DUMP on external table with add partition event throws NoSuchElementException (Sankar Hariappan, reviewed by Anishek Agarwal) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/21111654 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/21111654 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/21111654 Branch: refs/heads/master Commit: 2111165433d00900f437cd2d5e7893d0870754df Parents: b7723e4 Author: Sankar Hariappan <sank...@apache.org> Authored: Thu Sep 13 14:48:46 2018 +0530 Committer: Sankar Hariappan <sank...@apache.org> Committed: Thu Sep 13 14:48:46 2018 +0530 ---------------------------------------------------------------------- ...TestReplicationScenariosAcrossInstances.java | 21 ++++++++++++++++++++ .../repl/dump/events/AddPartitionHandler.java | 21 ++++++++++++-------- 2 files changed, 34 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/21111654/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java index 6c45641..35ad982 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java @@ -1400,4 +1400,25 @@ public class TestReplicationScenariosAcrossInstances { .run("select id from t4") .verifyResult(null); // Returns null as create table event doesn't list files } + + @Test + public void testDumpExternalTableWithAddPartitionEvent() throws Throwable { + WarehouseInstance.Tuple tuple = primary.dump("repl dump " + primaryDbName); + + replica.load(replicatedDbName, tuple.dumpLocation); + + tuple = primary.run("use " + primaryDbName) + .run("create external table t1 (place string) partitioned by (country string)") + .run("alter table t1 add partition(country='india')") + .run("alter table t1 add partition(country='us')") + .dump("repl dump " + primaryDbName + " from " + tuple.lastReplicationId + + " with ('hive.repl.include.external.tables'='true')"); + + replica.load(replicatedDbName, tuple.dumpLocation) + .run("use " + replicatedDbName) + .run("show tables like 't1'") + .verifyResult("t1") + .run("show partitions t1") + .verifyResults(new String[] { "country=india", "country=us" }); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/21111654/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java index cf15905..973a65b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java @@ -85,14 +85,19 @@ class AddPartitionHandler extends AbstractEventHandler { withinContext.hiveConf); Iterator<PartitionFiles> partitionFilesIter = apm.getPartitionFilesIter().iterator(); - for (Partition qlPtn : qlPtns) { - Iterable<String> files = partitionFilesIter.next().getFiles(); - if (files != null) { - // encoded filename/checksum of files, write into _files - try (BufferedWriter fileListWriter = writer(withinContext, qlPtn)) { - for (String file : files) { - fileListWriter.write(file); - fileListWriter.newLine(); + + // We expect one to one mapping between partitions and file iterators. For external table, this + // list would be empty. So, it is enough to check hasNext outside the loop. + if (partitionFilesIter.hasNext()) { + for (Partition qlPtn : qlPtns) { + Iterable<String> files = partitionFilesIter.next().getFiles(); + if (files != null) { + // encoded filename/checksum of files, write into _files + try (BufferedWriter fileListWriter = writer(withinContext, qlPtn)) { + for (String file : files) { + fileListWriter.write(file); + fileListWriter.newLine(); + } } } }
