Repository: hive Updated Branches: refs/heads/master c42bb86b8 -> 2a24612be
HIVE-14895 CompactorMR.CompactorOutputCommitter race condition (Eugene Koifman, reviewed by Wei Zheng) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e22e4112 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e22e4112 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e22e4112 Branch: refs/heads/master Commit: e22e411254dd21ab7df5ae619aebe50d68a07625 Parents: c42bb86 Author: Eugene Koifman <[email protected]> Authored: Tue Dec 6 16:21:54 2016 -0800 Committer: Eugene Koifman <[email protected]> Committed: Tue Dec 6 16:21:54 2016 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/e22e4112/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java index c3e3982..9ac2964 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java @@ -837,13 +837,15 @@ public class CompactorMR { @Override public void commitJob(JobContext context) throws IOException { JobConf conf = ShimLoader.getHadoopShims().getJobConf(context); - Path tmpLocation = new Path(conf.get(TMP_LOCATION)); + Path tmpLocation = new Path(conf.get(TMP_LOCATION));//this contains base_xxx or delta_xxx_yyy Path finalLocation = new Path(conf.get(FINAL_LOCATION)); FileSystem fs = tmpLocation.getFileSystem(conf); LOG.debug("Moving contents of " + tmpLocation.toString() + " to " + finalLocation.toString()); - FileStatus[] contents = fs.listStatus(tmpLocation); + FileStatus[] contents = fs.listStatus(tmpLocation);//expect 1 base or delta dir in this list + //we have MIN_TXN, MAX_TXN and IS_MAJOR in JobConf so we could figure out exactly what the dir + //name is that we want to rename; leave it for another day for (int i = 0; i < contents.length; i++) { Path newPath = new Path(finalLocation, contents[i].getPath().getName()); fs.rename(contents[i].getPath(), newPath);
