Author: prasadm
Date: Fri Jan 31 23:15:04 2014
New Revision: 1563334
URL: http://svn.apache.org/r1563334
Log:
HIVE-6209: 'LOAD DATA INPATH ... OVERWRITE ..' doesn't overwrite current data
(Szehon Ho via Prasad Mujumdar)
Added:
hive/trunk/ql/src/test/queries/clientpositive/load_fs_overwrite.q
hive/trunk/ql/src/test/results/clientpositive/load_fs_overwrite.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1563334&r1=1563333&r2=1563334&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Fri Jan
31 23:15:04 2014
@@ -2195,7 +2195,7 @@ private void constructOneLBLocationMap(F
}
}
success = fs.rename(srcf, destf);
- LOG.debug((replace ? "Replacing src:" : "Renaming src:") +
srcf.toString()
+ LOG.info((replace ? "Replacing src:" : "Renaming src:") + srcf.toString()
+ ";dest: " + destf.toString() + ";Status:" + success);
} catch (IOException ioe) {
throw new HiveException("Unable to move source" + srcf + " to
destination " + destf, ioe);
@@ -2297,10 +2297,7 @@ private void constructOneLBLocationMap(F
}
List<List<Path[]>> result = checkPaths(conf, fs, srcs, destf, true);
- // point of no return -- delete oldPath only if it is not same as destf,
- // otherwise, the oldPath/destf will be cleaned later just before move
- if (oldPath != null &&
(!destf.getFileSystem(conf).equals(oldPath.getFileSystem(conf))
- || !destf.equals(oldPath))) {
+ if (oldPath != null) {
try {
FileSystem fs2 = oldPath.getFileSystem(conf);
if (fs2.exists(oldPath)) {
@@ -2321,6 +2318,9 @@ private void constructOneLBLocationMap(F
Path destfp = destf.getParent();
if (!fs.exists(destfp)) {
boolean success = fs.mkdirs(destfp);
+ if (!success) {
+ LOG.warn("Error creating directory " + destf.toString());
+ }
if (inheritPerms && success) {
fs.setPermission(destfp,
fs.getFileStatus(destfp.getParent()).getPermission());
}
@@ -2334,6 +2334,9 @@ private void constructOneLBLocationMap(F
} else { // srcf is a file or pattern containing wildcards
if (!fs.exists(destf)) {
boolean success = fs.mkdirs(destf);
+ if (!success) {
+ LOG.warn("Error creating directory " + destf.toString());
+ }
if (inheritPerms && success) {
fs.setPermission(destf,
fs.getFileStatus(destf.getParent()).getPermission());
}
Added: hive/trunk/ql/src/test/queries/clientpositive/load_fs_overwrite.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/load_fs_overwrite.q?rev=1563334&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/load_fs_overwrite.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/load_fs_overwrite.q Fri Jan
31 23:15:04 2014
@@ -0,0 +1,20 @@
+--HIVE 6209
+
+drop table target;
+drop table temp;
+
+create table target (key string, value string) stored as textfile location
'file:${system:test.tmp.dir}/target';
+create table temp (key string, value string) stored as textfile location
'file:${system:test.tmp.dir}/temp';
+
+set fs.pfile.impl.disable.cache=false;
+
+load data local inpath '../../data/files/kv1.txt' into table temp;
+load data inpath '${system:test.tmp.dir}/temp/kv1.txt' overwrite into table
target;
+select count(*) from target;
+
+load data local inpath '../../data/files/kv2.txt' into table temp;
+load data inpath '${system:test.tmp.dir}/temp/kv2.txt' overwrite into table
target;
+select count(*) from target;
+
+drop table target;
+drop table temp;
\ No newline at end of file
Added: hive/trunk/ql/src/test/results/clientpositive/load_fs_overwrite.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/load_fs_overwrite.q.out?rev=1563334&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/load_fs_overwrite.q.out
(added)
+++ hive/trunk/ql/src/test/results/clientpositive/load_fs_overwrite.q.out Fri
Jan 31 23:15:04 2014
@@ -0,0 +1,80 @@
+PREHOOK: query: --HIVE 6209
+
+drop table target
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: --HIVE 6209
+
+drop table target
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table temp
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table temp
+POSTHOOK: type: DROPTABLE
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@target
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@temp
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table
temp
+PREHOOK: type: LOAD
+PREHOOK: Output: default@temp
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table
temp
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@temp
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+PREHOOK: Output: default@target
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@target
+PREHOOK: query: select count(*) from target
+PREHOOK: type: QUERY
+PREHOOK: Input: default@target
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from target
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@target
+#### A masked pattern was here ####
+500
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table
temp
+PREHOOK: type: LOAD
+PREHOOK: Output: default@temp
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table
temp
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@temp
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+PREHOOK: Output: default@target
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@target
+PREHOOK: query: select count(*) from target
+PREHOOK: type: QUERY
+PREHOOK: Input: default@target
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from target
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@target
+#### A masked pattern was here ####
+500
+PREHOOK: query: drop table target
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@target
+PREHOOK: Output: default@target
+POSTHOOK: query: drop table target
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@target
+POSTHOOK: Output: default@target
+PREHOOK: query: drop table temp
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@temp
+PREHOOK: Output: default@temp
+POSTHOOK: query: drop table temp
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@temp
+POSTHOOK: Output: default@temp
