zhangbutao commented on PR #5407:
URL: https://github.com/apache/hive/pull/5407#issuecomment-2550468233
I changed the `MoveTask.java`, to skip create HDFS directory
`/path/to/local/dir` if targetPath is local.
So, you can change your PR like this :
```
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index 2721977d6f..6ab137b418 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -438,9 +438,11 @@ public int execute() {
}
}
else {
- FileSystem targetFs = targetPath.getFileSystem(conf);
- if (!targetFs.exists(targetPath.getParent())){
- targetFs.mkdirs(targetPath.getParent());
+ if (lfd.getIsDfsDir()) {
+ FileSystem targetFs = targetPath.getFileSystem(conf);
+ if (!targetFs.exists(targetPath.getParent())) {
+ targetFs.mkdirs(targetPath.getParent());
+ }
}
moveFile(sourcePath, targetPath, lfd.getIsDfsDir());
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 5f81157124..04ddb857c0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -8468,6 +8468,7 @@ private FileSinkDesc createFileSinkDesc(String dest,
TableDesc table_desc,
RowSchema fsRS, boolean
canBeMerged, Table dest_tab, boolean isMmCtas,
Integer dest_type, QB qb, boolean
isDirectInsert, AcidUtils.Operation acidOperation, String moveTaskId) throws
SemanticException {
boolean isInsertOverwrite = false;
+ boolean isLocal = false;
Context.Operation writeOperation = getWriteOperation(dest);
switch (dest_type) {
case QBMetaData.DEST_PARTITION:
@@ -8492,6 +8493,7 @@ private FileSinkDesc createFileSinkDesc(String dest,
TableDesc table_desc,
break;
case QBMetaData.DEST_LOCAL_FILE:
+ isLocal = true;
case QBMetaData.DEST_DFS_FILE:
//CTAS path or insert into file/directory
break;
@@ -8545,7 +8547,12 @@ private FileSinkDesc createFileSinkDesc(String dest,
TableDesc table_desc,
fileSinkDesc.setStatsAggPrefix(fileSinkDesc.getDirName().toString());
if (!destTableIsMaterialization &&
HiveConf.getVar(conf,
HIVE_STATS_DBCLASS).equalsIgnoreCase(StatDB.fs.name())) {
- String statsTmpLoc =
ctx.getTempDirForInterimJobPath(dest_path).toString();
+ String statsTmpLoc;
+ if (isLocal){
+ statsTmpLoc = ctx.getMRTmpPath().toString();
+ } else {
+ statsTmpLoc = ctx.getTempDirForInterimJobPath(dest_path).toString();
+ }
fileSinkDesc.setStatsTmpDir(statsTmpLoc);
LOG.debug("Set stats collection dir : " + statsTmpLoc);
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: gitbox-unsubscr...@hive.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscr...@hive.apache.org
For additional commands, e-mail: gitbox-h...@hive.apache.org
