This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new fed4a68 [SPARK-31163][SQL] TruncateTableCommand with acl/permission
should handle non-existed path
fed4a68 is described below
commit fed4a680e3c45462d0d69aeac53780a677f988ae
Author: yi.wu <[email protected]>
AuthorDate: Mon Mar 16 11:45:25 2020 -0700
[SPARK-31163][SQL] TruncateTableCommand with acl/permission should handle
non-existed path
### What changes were proposed in this pull request?
This fix #26956
Wrap try-catch on `fs.getFileStatus(path)` within acl/permission in case of
the path doesn't exist.
### Why are the changes needed?
`truncate table` may fail to re-create path in case of interruption or
something else. As a result, next time we `truncate table` on the same table
with acl/permission, it will fail due to `FileNotFoundException`. And it also
brings behavior change compares to previous Spark version, which could still
`truncate table` successfully even if the path doesn't exist.
### Does this PR introduce any user-facing change?
No.
### How was this patch tested?
Added UT.
Closes #27923 from Ngone51/fix_truncate.
Authored-by: yi.wu <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit cb26f636b08aea4c5c6bf5035a359cd3cbf335c0)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../apache/spark/sql/execution/command/tables.scala | 2 +-
.../spark/sql/execution/command/DDLSuite.scala | 21 +++++++++++++++++++++
2 files changed, 22 insertions(+), 1 deletion(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index 6243261..d4de822 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -508,8 +508,8 @@ case class TruncateTableCommand(
var optPermission: Option[FsPermission] = None
var optAcls: Option[java.util.List[AclEntry]] = None
if (!ignorePermissionAcl) {
- val fileStatus = fs.getFileStatus(path)
try {
+ val fileStatus = fs.getFileStatus(path)
optPermission = Some(fileStatus.getPermission())
} catch {
case NonFatal(_) => // do nothing
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index 5a67dce..10ad8ac 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -2084,6 +2084,27 @@ abstract class DDLSuite extends QueryTest with
SQLTestUtils {
}
}
+ test("SPARK-31163: acl/permission should handle non-existed path when
truncating table") {
+ withSQLConf(SQLConf.TRUNCATE_TABLE_IGNORE_PERMISSION_ACL.key -> "false") {
+ withTable("tab1") {
+ sql("CREATE TABLE tab1 (col1 STRING, col2 INT) USING parquet
PARTITIONED BY (col2)")
+ sql("INSERT INTO tab1 SELECT 'one', 1")
+ checkAnswer(spark.table("tab1"), Row("one", 1))
+ val part =
spark.sessionState.catalog.listPartitions(TableIdentifier("tab1")).head
+ val path = new File(part.location.getPath)
+ sql("TRUNCATE TABLE tab1")
+ // simulate incomplete/unsuccessful truncate
+ assert(path.exists())
+ path.delete()
+ assert(!path.exists())
+ // execute without java.io.FileNotFoundException
+ sql("TRUNCATE TABLE tab1")
+ // partition path should be re-created
+ assert(path.exists())
+ }
+ }
+ }
+
test("create temporary view with mismatched schema") {
withTable("tab1") {
spark.range(10).write.saveAsTable("tab1")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
