AngersZhuuuu commented on a change in pull request #29319:
URL: https://github.com/apache/spark/pull/29319#discussion_r482796419
##########
File path:
sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
##########
@@ -309,9 +309,23 @@ case class InsertIntoHiveTable(
partitionPath.foreach { path =>
val fs = path.getFileSystem(hadoopConf)
if (fs.exists(path)) {
- if (!fs.delete(path, true)) {
- throw new RuntimeException(
- s"Cannot remove partition directory '$path'")
+ val isTrashEnabled =
sparkSession.sessionState.conf.trashEnabled
+ if (!isTrashEnabled) {
+ if (!fs.delete(path, true)) {
+ throw new RuntimeException(
+ s"Cannot remove partition directory '$path'")
+ }
+ } else {
+ logDebug(s"Try to move data ${path.toString} to trash")
+ val isSuccess = Trash.moveToAppropriateTrash(fs, path,
hadoopConf)
+ if (!isSuccess) {
+ logWarning(s"Failed to move data ${path.toString} to trash
" +
+ "fallback to hard deletion")
+ if (!fs.delete(path, true)) {
+ throw new RuntimeException(
+ s"Cannot remove partition directory '$path'")
+ }
+ }
Review comment:
There are too many case
1. Hive version > 2.0 & Hive version < 2.9
2. insert overwrite table
3. dynamic partition insert
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
