Repository: hive Updated Branches: refs/heads/branch-3 2615d98ef -> ab6d891f1
HIVE-19970: Replication dump has a NPE when table is empty (Mahesh Kumar Behera, reviewed by Peter Vary, Sankar Hariappan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ab6d891f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ab6d891f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ab6d891f Branch: refs/heads/branch-3 Commit: ab6d891f1dbc78d1c7110dd80aa48807c57ec41e Parents: 2615d98 Author: Sankar Hariappan <[email protected]> Authored: Tue Jul 10 10:17:22 2018 +0530 Committer: Sankar Hariappan <[email protected]> Committed: Tue Jul 10 10:17:22 2018 +0530 ---------------------------------------------------------------------- .../hive/ql/parse/TestReplicationScenarios.java | 42 ++++++++++++++++ .../TestReplicationScenariosAcidTables.java | 52 +++++++++++++++++++- ...TestReplicationScenariosAcrossInstances.java | 7 ++- .../hadoop/hive/ql/parse/WarehouseInstance.java | 7 ++- .../org/apache/hadoop/hive/ql/ErrorMsg.java | 3 +- .../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 4 ++ .../hadoop/hive/ql/exec/repl/ReplUtils.java | 7 ++- .../exec/repl/bootstrap/load/LoadDatabase.java | 2 +- .../ql/parse/repl/dump/PartitionExport.java | 24 ++++++--- .../hadoop/hive/ql/parse/repl/dump/Utils.java | 8 ++- .../ql/parse/repl/dump/io/FileOperations.java | 7 +++ 11 files changed, 147 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/ab6d891f/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index d161841..d1f0def 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -90,6 +90,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.apache.hadoop.hive.metastore.ReplChangeManager.SOURCE_OF_REPLICATION; +import org.junit.Assert; public class TestReplicationScenarios { @@ -3185,6 +3186,47 @@ public class TestReplicationScenarios { } @Test + public void testDumpWithTableDirMissing() throws IOException { + String dbName = createDB(testName.getMethodName(), driver); + run("CREATE TABLE " + dbName + ".normal(a int)", driver); + run("INSERT INTO " + dbName + ".normal values (1)", driver); + + Path path = new Path(System.getProperty("test.warehouse.dir", "")); + path = new Path(path, dbName.toLowerCase() + ".db"); + path = new Path(path, "normal"); + FileSystem fs = path.getFileSystem(hconf); + fs.delete(path); + + advanceDumpDir(); + CommandProcessorResponse ret = driver.run("REPL DUMP " + dbName); + Assert.assertEquals(ret.getResponseCode(), ErrorMsg.FILE_NOT_FOUND.getErrorCode()); + + run("DROP TABLE " + dbName + ".normal", driver); + run("drop database " + dbName, true, driver); + } + + @Test + public void testDumpWithPartitionDirMissing() throws IOException { + String dbName = createDB(testName.getMethodName(), driver); + run("CREATE TABLE " + dbName + ".normal(a int) PARTITIONED BY (part int)", driver); + run("INSERT INTO " + dbName + ".normal partition (part= 124) values (1)", driver); + + Path path = new Path(System.getProperty("test.warehouse.dir","")); + path = new Path(path, dbName.toLowerCase()+".db"); + path = new Path(path, "normal"); + path = new Path(path, "part=124"); + FileSystem fs = path.getFileSystem(hconf); + fs.delete(path); + + advanceDumpDir(); + CommandProcessorResponse ret = driver.run("REPL DUMP " + dbName); + Assert.assertEquals(ret.getResponseCode(), ErrorMsg.FILE_NOT_FOUND.getErrorCode()); + + run("DROP TABLE " + dbName + ".normal", driver); + run("drop database " + dbName, true, driver); + } + + @Test public void testDumpNonReplDatabase() throws IOException { String dbName = createDBNonRepl(testName.getMethodName(), driver); verifyFail("REPL DUMP " + dbName, driver); http://git-wip-us.apache.org/repos/asf/hive/blob/ab6d891f/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java index a1498ca..3ee0747 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java @@ -32,6 +32,10 @@ import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore; import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.CallerArguments; import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection; import static org.apache.hadoop.hive.metastore.ReplChangeManager.SOURCE_OF_REPLICATION; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.junit.rules.TestName; import org.junit.rules.TestRule; import org.slf4j.Logger; @@ -63,10 +67,11 @@ public class TestReplicationScenariosAcidTables { protected static final Logger LOG = LoggerFactory.getLogger(TestReplicationScenarios.class); private static WarehouseInstance primary, replica, replicaNonAcid; private String primaryDbName, replicatedDbName; + private static HiveConf conf; @BeforeClass public static void classLevelSetup() throws Exception { - Configuration conf = new Configuration(); + conf = new HiveConf(TestReplicationScenariosAcidTables.class); conf.set("dfs.client.use.datanode.hostname", "true"); conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); MiniDFSCluster miniDFSCluster = @@ -438,4 +443,49 @@ public class TestReplicationScenariosAcidTables { .run("select name from t2 order by name") .verifyResults(Arrays.asList("bob", "carl")); } + + @Test + public void testDumpAcidTableWithPartitionDirMissing() throws Throwable { + String dbName = testName.getMethodName(); + primary.run("CREATE DATABASE " + dbName + " WITH DBPROPERTIES ( '" + + SOURCE_OF_REPLICATION + "' = '1,2,3')") + .run("CREATE TABLE " + dbName + ".normal (a int) PARTITIONED BY (part int)" + + " STORED AS ORC TBLPROPERTIES ('transactional'='true')") + .run("INSERT INTO " + dbName + ".normal partition (part= 124) values (1)"); + + Path path = new Path(primary.warehouseRoot, dbName.toLowerCase()+".db"); + path = new Path(path, "normal"); + path = new Path(path, "part=124"); + FileSystem fs = path.getFileSystem(conf); + fs.delete(path); + + CommandProcessorResponse ret = primary.runCommand("REPL DUMP " + dbName + + " with ('hive.repl.dump.include.acid.tables' = 'true')"); + Assert.assertEquals(ret.getResponseCode(), ErrorMsg.FILE_NOT_FOUND.getErrorCode()); + + primary.run("DROP TABLE " + dbName + ".normal"); + primary.run("drop database " + dbName); + } + + @Test + public void testDumpAcidTableWithTableDirMissing() throws Throwable { + String dbName = testName.getMethodName(); + primary.run("CREATE DATABASE " + dbName + " WITH DBPROPERTIES ( '" + + SOURCE_OF_REPLICATION + "' = '1,2,3')") + .run("CREATE TABLE " + dbName + ".normal (a int) " + + " STORED AS ORC TBLPROPERTIES ('transactional'='true')") + .run("INSERT INTO " + dbName + ".normal values (1)"); + + Path path = new Path(primary.warehouseRoot, dbName.toLowerCase()+".db"); + path = new Path(path, "normal"); + FileSystem fs = path.getFileSystem(conf); + fs.delete(path); + + CommandProcessorResponse ret = primary.runCommand("REPL DUMP " + dbName + + " with ('hive.repl.dump.include.acid.tables' = 'true')"); + Assert.assertEquals(ret.getResponseCode(), ErrorMsg.FILE_NOT_FOUND.getErrorCode()); + + primary.run("DROP TABLE " + dbName + ".normal"); + primary.run("drop database " + dbName); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/ab6d891f/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java index c5f35e2..fbd893b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java @@ -66,6 +66,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.apache.hadoop.hive.metastore.ReplChangeManager.SOURCE_OF_REPLICATION; +import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.junit.Assert; public class TestReplicationScenariosAcrossInstances { @Rule @@ -1096,7 +1099,9 @@ public class TestReplicationScenariosAcrossInstances { assertEquals(0, replica.getForeignKeyList(replicatedDbName, "t2").size()); // Retry with different dump should fail. - replica.loadFailure(replicatedDbName, tuple2.dumpLocation); + CommandProcessorResponse ret = replica.runCommand("REPL LOAD " + replicatedDbName + + " FROM '" + tuple2.dumpLocation + "'"); + Assert.assertEquals(ret.getResponseCode(), ErrorMsg.REPL_BOOTSTRAP_LOAD_PATH_NOT_VALID.getErrorCode()); // Verify if create table is not called on table t1 but called for t2 and t3. // Also, allow constraint creation only on t1 and t3. Foreign key creation on t2 fails. http://git-wip-us.apache.org/repos/asf/hive/blob/ab6d891f/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java index 26f488e..e9875b4 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java @@ -77,6 +77,7 @@ public class WarehouseInstance implements Closeable { HiveConf hiveConf; MiniDFSCluster miniDFSCluster; private HiveMetaStoreClient client; + public final Path warehouseRoot; private static int uniqueIdentifier = 0; @@ -90,7 +91,7 @@ public class WarehouseInstance implements Closeable { assert miniDFSCluster.isDataNodeUp(); DistributedFileSystem fs = miniDFSCluster.getFileSystem(); - Path warehouseRoot = mkDir(fs, "/warehouse" + uniqueIdentifier); + warehouseRoot = mkDir(fs, "/warehouse" + uniqueIdentifier); if (StringUtils.isNotEmpty(keyNameForEncryptedZone)) { fs.createEncryptionZone(warehouseRoot, keyNameForEncryptedZone); } @@ -200,6 +201,10 @@ public class WarehouseInstance implements Closeable { return this; } + public CommandProcessorResponse runCommand(String command) throws Throwable { + return driver.run(command); + } + WarehouseInstance runFailure(String command) throws Throwable { CommandProcessorResponse ret = driver.run(command); if (ret.getException() == null) { http://git-wip-us.apache.org/repos/asf/hive/blob/ab6d891f/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index e0d336a..7bea5e4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -502,7 +502,8 @@ public enum ErrorMsg { //if the error message is changed for REPL_EVENTS_MISSING_IN_METASTORE, then need modification in getNextNotification //method in HiveMetaStoreClient REPL_EVENTS_MISSING_IN_METASTORE(20016, "Notification events are missing in the meta store."), - REPL_BOOTSTRAP_LOAD_PATH_NOT_VALID(20017, "Target database is bootstrapped from some other path."), + REPL_BOOTSTRAP_LOAD_PATH_NOT_VALID(20017, "Load path {0} not valid as target database is bootstrapped " + + "from some other path : {1}."), REPL_FILE_MISSING_FROM_SRC_AND_CM_PATH(20018, "File is missing from both source and cm path."), REPL_LOAD_PATH_NOT_FOUND(20019, "Load path does not exist."), REPL_DATABASE_IS_NOT_SOURCE_OF_REPLICATION(20020, http://git-wip-us.apache.org/repos/asf/hive/blob/ab6d891f/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java index 78a0ba3..a14802f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java @@ -121,6 +121,10 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable { lastReplId = incrementalDump(dumpRoot, dmd, cmRoot); } prepareReturnValues(Arrays.asList(dumpRoot.toUri().toString(), String.valueOf(lastReplId)), dumpSchema); + } catch (RuntimeException e) { + LOG.error("failed", e); + setException(e); + return ErrorMsg.getErrorMsg(e.getMessage()).getErrorCode(); } catch (Exception e) { LOG.error("failed", e); setException(e); http://git-wip-us.apache.org/repos/asf/hive/blob/ab6d891f/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplUtils.java index 18a8304..e41e4b5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplUtils.java @@ -16,7 +16,7 @@ * limitations under the License. */ package org.apache.hadoop.hive.ql.exec.repl; - +import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.ql.exec.Task; @@ -114,9 +114,8 @@ public class ReplUtils { if (props.get(REPL_CHECKPOINT_KEY).equals(dumpRoot)) { return true; } - throw new InvalidOperationException("REPL LOAD with Dump: " + dumpRoot - + " is not allowed as the target DB: " + dbName - + " is already bootstrap loaded by another Dump " + props.get(REPL_CHECKPOINT_KEY)); + throw new InvalidOperationException(ErrorMsg.REPL_BOOTSTRAP_LOAD_PATH_NOT_VALID.format(dumpRoot, + props.get(REPL_CHECKPOINT_KEY))); } return false; } http://git-wip-us.apache.org/repos/asf/hive/blob/ab6d891f/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java index 0270d2a..7ddae6f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java @@ -78,7 +78,7 @@ public class LoadDatabase { } return tracker; } catch (Exception e) { - throw new SemanticException(e); + throw new SemanticException(e.getMessage(), e); } } http://git-wip-us.apache.org/repos/asf/hive/blob/ab6d891f/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/PartitionExport.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/PartitionExport.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/PartitionExport.java index d73fc4f..4ed6b37 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/PartitionExport.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/PartitionExport.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.parse.repl.dump; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.PartitionIterable; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; @@ -36,6 +37,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.LinkedList; +import java.util.concurrent.Future; import static org.apache.hadoop.hive.ql.parse.repl.dump.TableExport.Paths; @@ -70,10 +73,11 @@ class PartitionExport { this.callersSession = SessionState.get(); } - void write(final ReplicationSpec forReplicationSpec) throws InterruptedException { + void write(final ReplicationSpec forReplicationSpec) throws InterruptedException, HiveException { + List<Future<?>> futures = new LinkedList<>(); ExecutorService producer = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setNameFormat("partition-submitter-thread-%d").build()); - producer.submit(() -> { + futures.add(producer.submit(() -> { SessionState.setCurrentSessionState(callersSession); for (Partition partition : partitionIterable) { try { @@ -83,7 +87,7 @@ class PartitionExport { "Error while queuing up the partitions for export of data files", e); } } - }); + })); producer.shutdown(); ThreadFactory namingThreadFactory = @@ -102,7 +106,7 @@ class PartitionExport { continue; } LOG.debug("scheduling partition dump {}", partition.getName()); - consumer.submit(() -> { + futures.add(consumer.submit(() -> { String partitionName = partition.getName(); String threadName = Thread.currentThread().getName(); LOG.debug("Thread: {}, start partition dump {}", threadName, partitionName); @@ -115,11 +119,19 @@ class PartitionExport { .export(forReplicationSpec); LOG.debug("Thread: {}, finish partition dump {}", threadName, partitionName); } catch (Exception e) { - throw new RuntimeException("Error while export of data files", e); + throw new RuntimeException(e.getMessage(), e); } - }); + })); } consumer.shutdown(); + for (Future<?> future : futures) { + try { + future.get(); + } catch (Exception e) { + LOG.error("failed", e.getCause()); + throw new HiveException(e.getCause().getMessage(), e.getCause()); + } + } // may be drive this via configuration as well. consumer.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS); } http://git-wip-us.apache.org/repos/asf/hive/blob/ab6d891f/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java index 2df684d..c0701c5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.AcidUtils; @@ -38,6 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.DataOutputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.Collection; import java.util.Collections; @@ -210,7 +212,11 @@ public class Utils { static List<Path> getDataPathList(Path fromPath, ReplicationSpec replicationSpec, HiveConf conf) throws IOException { if (replicationSpec.isTransactionalTableDump()) { - return AcidUtils.getValidDataPaths(fromPath, conf, replicationSpec.getValidWriteIdList()); + try { + return AcidUtils.getValidDataPaths(fromPath, conf, replicationSpec.getValidWriteIdList()); + } catch (FileNotFoundException e) { + throw new IOException(ErrorMsg.FILE_NOT_FOUND.format(e.getMessage()), e); + } } else { return Collections.singletonList(fromPath); } http://git-wip-us.apache.org/repos/asf/hive/blob/ab6d891f/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java index 070c830..a934d81 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.parse.repl.dump.io; import java.io.BufferedWriter; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStreamWriter; import java.util.ArrayList; @@ -46,6 +47,8 @@ import org.apache.hadoop.hive.shims.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.hive.ql.ErrorMsg.FILE_NOT_FOUND; + //TODO: this object is created once to call one method and then immediately destroyed. //So it's basically just a roundabout way to pass arguments to a static method. Simplify? public class FileOperations { @@ -158,6 +161,10 @@ public class FileOperations { } done = true; } catch (IOException e) { + if (e instanceof FileNotFoundException) { + logger.error("exporting data files in dir : " + dataPathList + " to " + exportRootDataDir + " failed"); + throw new FileNotFoundException(FILE_NOT_FOUND.format(e.getMessage())); + } repeat++; logger.info("writeFilesList failed", e); if (repeat >= FileUtils.MAX_IO_ERROR_RETRY) {
