HIVE-14614: Insert overwrite local directory fails with IllegalStateException (Vihang Karajgaonkar, reviewed by Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/673f3263 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/673f3263 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/673f3263 Branch: refs/heads/hive-14535 Commit: 673f3263af8dd8c1cc890cb159b6328e4eda5694 Parents: 67bf8eb Author: Vihang Karajgaonkar <[email protected]> Authored: Mon Aug 29 15:48:01 2016 -0500 Committer: Sergio Pena <[email protected]> Committed: Mon Aug 29 15:48:01 2016 -0500 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hive/ql/Context.java | 19 ++++++++++++++++++- .../apache/hadoop/hive/ql/exec/Utilities.java | 1 + .../apache/hadoop/hive/ql/exec/TestContext.java | 5 +++++ 3 files changed, 24 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/673f3263/ql/src/java/org/apache/hadoop/hive/ql/Context.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java index 3785b1e..4667f68 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.BlobStorageUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.TaskRunner; +import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager.Heartbeater; @@ -361,7 +362,9 @@ public class Context { * @return A path to the new temporary directory */ public Path getTempDirForPath(Path path) { - if (BlobStorageUtils.isBlobStoragePath(conf, path) && !BlobStorageUtils.isBlobStorageAsScratchDir(conf)) { + boolean isLocal = isPathLocal(path); + if ((BlobStorageUtils.isBlobStoragePath(conf, path) && !BlobStorageUtils.isBlobStorageAsScratchDir(conf)) + || isLocal) { // For better write performance, we use HDFS for temporary data when object store is used. // Note that the scratch directory configuration variable must use HDFS or any other non-blobstorage system // to take advantage of this performance. @@ -371,6 +374,20 @@ public class Context { } } + /* + * Checks if the path is for the local filesystem or not + */ + private boolean isPathLocal(Path path) { + boolean isLocal = false; + if (path != null) { + String scheme = path.toUri().getScheme(); + if (scheme != null) { + isLocal = scheme.equals(Utilities.HADOOP_LOCAL_FS_SCHEME); + } + } + return isLocal; + } + private Path getExternalScratchDir(URI extURI) { return getStagingDir(new Path(extURI.getScheme(), extURI.getAuthority(), extURI.getPath()), !explain); } http://git-wip-us.apache.org/repos/asf/hive/blob/673f3263/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index a542dc4..b4c6982 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -213,6 +213,7 @@ public final class Utilities { */ public static String HADOOP_LOCAL_FS = "file:///"; + public static final String HADOOP_LOCAL_FS_SCHEME = "file"; public static String MAP_PLAN_NAME = "map.xml"; public static String REDUCE_PLAN_NAME = "reduce.xml"; public static String MERGE_PLAN_NAME = "merge.xml"; http://git-wip-us.apache.org/repos/asf/hive/blob/673f3263/ql/src/test/org/apache/hadoop/hive/ql/exec/TestContext.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestContext.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestContext.java index 4a4c240..808cb94 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestContext.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestContext.java @@ -54,6 +54,11 @@ public class TestContext { doReturn(mrTmpPath).when(spyContext).getMRTmpPath(); assertEquals(mrTmpPath, spyContext.getTempDirForPath(new Path("s3a://bucket/dir"))); + // When local filesystem paths are used, then getMRTmpPatch() should be called to + // get a temporary directory + assertEquals(mrTmpPath, spyContext.getTempDirForPath(new Path("file:/user"))); + assertEquals(mrTmpPath, spyContext.getTempDirForPath(new Path("file:///user"))); + // When Non-Object store paths are used, then getExtTmpPathRelTo is called to get a temporary // directory on the same path passed as a parameter Path tmpPathRelTo = new Path("hdfs://hostname/user");
