HIVE-16156: FileSinkOperator should delete existing output target when renaming (Reviewed by Sergey)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/76b65baa Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/76b65baa Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/76b65baa Branch: refs/heads/hive-14535 Commit: 76b65baa7b5f5c0e5c1f99cf0621247f65fb0b00 Parents: 8dda898 Author: Xuefu Zhang <xu...@uber.com> Authored: Fri Mar 10 22:14:49 2017 -0800 Committer: Xuefu Zhang <xu...@uber.com> Committed: Fri Mar 10 22:14:49 2017 -0800 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/exec/FileSinkOperator.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/76b65baa/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index 3bbe92d..a9d03d0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -229,8 +229,18 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements } } if (needToRename && outPaths[idx] != null && !fs.rename(outPaths[idx], finalPaths[idx])) { - throw new HiveException("Unable to rename output from: " + + FileStatus fileStatus = FileUtils.getFileStatusOrNull(fs, finalPaths[idx]); + if (fileStatus != null) { + LOG.warn("Target path " + finalPaths[idx] + " with a size " + fileStatus.getLen() + " exists. Trying to delete it."); + if (!fs.delete(finalPaths[idx], true)) { + throw new HiveException("Unable to delete existing target output: " + finalPaths[idx]); + } + } + + if (!fs.rename(outPaths[idx], finalPaths[idx])) { + throw new HiveException("Unable to rename output from: " + outPaths[idx] + " to: " + finalPaths[idx]); + } } updateProgress(); } catch (IOException e) {
