Github user yhuai commented on a diff in the pull request:
https://github.com/apache/spark/pull/13315#discussion_r64794004
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala ---
@@ -289,37 +289,53 @@ case class TruncateTableCommand(
val catalog = sparkSession.sessionState.catalog
if (!catalog.tableExists(tableName)) {
throw new AnalysisException(s"Table '$tableName' in TRUNCATE TABLE
does not exist.")
- } else if (catalog.isTemporaryTable(tableName)) {
+ }
+ if (catalog.isTemporaryTable(tableName)) {
throw new AnalysisException(
s"Operation not allowed: TRUNCATE TABLE on temporary tables:
'$tableName'")
+ }
+ val table = catalog.getTableMetadata(tableName)
+ if (table.tableType == CatalogTableType.EXTERNAL) {
+ throw new AnalysisException(
+ s"Operation not allowed: TRUNCATE TABLE on external tables:
'$tableName'")
+ }
+ if (table.tableType == CatalogTableType.VIEW) {
+ throw new AnalysisException(
+ s"Operation not allowed: TRUNCATE TABLE on views: '$tableName'")
+ }
+ if (DDLUtils.isDatasourceTable(table) && partitionSpec.isDefined) {
+ throw new AnalysisException(
+ s"Operation not allowed: TRUNCATE TABLE ... PARTITION is not
supported " +
+ s"for tables created using the data sources API: '$tableName'")
+ }
+ val locations = if (partitionSpec.isDefined) {
+ catalog.listPartitions(tableName,
partitionSpec).map(_.storage.locationUri)
} else {
- val locations = if (partitionSpec.isDefined) {
- catalog.listPartitions(tableName,
partitionSpec).map(_.storage.locationUri)
+ if (table.partitionColumnNames.nonEmpty) {
+ catalog.listPartitions(tableName).map(_.storage.locationUri)
} else {
- val table = catalog.getTableMetadata(tableName)
- if (table.partitionColumnNames.nonEmpty) {
- catalog.listPartitions(tableName).map(_.storage.locationUri)
- } else {
- Seq(table.storage.locationUri)
- }
+ Seq(table.storage.locationUri)
}
- val hadoopConf = sparkSession.sessionState.newHadoopConf()
- locations.foreach { location =>
- if (location.isDefined) {
- val path = new Path(location.get)
- try {
- val fs = path.getFileSystem(hadoopConf)
- fs.delete(path, true)
- fs.mkdirs(path)
- } catch {
- case NonFatal(e) =>
- throw new AnalysisException(
- s"Failed to truncate table '$tableName' when removing data
of the path: $path " +
- s"because of ${e.toString}")
- }
+ }
+ val hadoopConf = sparkSession.sessionState.newHadoopConf()
+ locations.foreach { location =>
+ if (location.isDefined) {
+ val path = new Path(location.get)
+ try {
+ val fs = path.getFileSystem(hadoopConf)
+ fs.delete(path, true)
+ fs.mkdirs(path)
+ } catch {
+ case NonFatal(e) =>
+ throw new AnalysisException(
+ s"Failed to truncate table '$tableName' when removing data
of the path: $path " +
+ s"because of ${e.toString}")
}
}
}
+ // After deleting the data, invalidate the table to make sure we don't
keep around a stale
+ // file relation in the metastore cache.
+ sparkSession.sessionState.invalidateTable(tableName.unquotedString)
--- End diff --
Seems we also need to drop the cached data in the InMemoryColumnarStore?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
