This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b71b917 [SPARK-38236][SQL] Treat table location as absolute when the
first letter of its path is slash in create/alter table
b71b917 is described below
commit b71b917ddfe60f93f096e54aedf5ad6474cffe7a
Author: Bo Zhang <[email protected]>
AuthorDate: Mon Feb 21 11:30:28 2022 +0800
[SPARK-38236][SQL] Treat table location as absolute when the first letter
of its path is slash in create/alter table
### What changes were proposed in this pull request?
After https://github.com/apache/spark/pull/28527, we change to create table
under the database location when the table location is relative. However the
criteria to determine if a table location is relative/absolute is
`URI.isAbsolute`, which basically checks if the table location URI has a scheme
defined. So table URIs like `/table/path` are treated as relative and the
scheme and authority of the database location URI are used to create the table.
For example, when the database locat [...]
This change fixes that by treating table location as absolute when the
first letter of its path is slash.
This also applies to alter table.
### Why are the changes needed?
This is to fix the behavior described above.
### Does this PR introduce _any_ user-facing change?
Yes. When users try to create/alter a table with a location that starts
with a slash but without a scheme defined, the table will be created
under/altered to the file system defined in `SessionCatalog.hadoopConf`,
instead of the one defined in the database location URI.
### How was this patch tested?
Updated unit tests.
Closes #35462 from bozhang2820/spark-31709.
Authored-by: Bo Zhang <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../org/apache/spark/sql/avro/AvroSuite.scala | 2 ++
.../spark/mllib/util/MLlibTestSparkContext.scala | 2 ++
.../sql/catalyst/catalog/SessionCatalog.scala | 2 ++
.../sql-tests/results/show-create-table.sql.out | 4 ++--
.../spark/sql/connector/DataSourceV2SQLSuite.scala | 2 +-
.../command/v1/ShowCreateTableSuite.scala | 2 +-
.../datasources/v2/V2SessionCatalogSuite.scala | 25 ++++++++++++++++------
7 files changed, 29 insertions(+), 10 deletions(-)
diff --git
a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index d9d8c3c..05d57ec 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -68,6 +68,8 @@ abstract class AvroSuite
override protected def beforeAll(): Unit = {
super.beforeAll()
+ // initialize SessionCatalog here so it has a clean hadoopConf
+ spark.sessionState.catalog
spark.conf.set(SQLConf.FILES_MAX_PARTITION_BYTES.key, 1024)
}
diff --git
a/mllib/src/test/scala/org/apache/spark/mllib/util/MLlibTestSparkContext.scala
b/mllib/src/test/scala/org/apache/spark/mllib/util/MLlibTestSparkContext.scala
index 5eb128a..3a7040d 100644
---
a/mllib/src/test/scala/org/apache/spark/mllib/util/MLlibTestSparkContext.scala
+++
b/mllib/src/test/scala/org/apache/spark/mllib/util/MLlibTestSparkContext.scala
@@ -40,6 +40,8 @@ trait MLlibTestSparkContext extends TempDirectory { self:
Suite =>
.appName("MLlibUnitTest")
.getOrCreate()
sc = spark.sparkContext
+ // initialize SessionCatalog here so it has a clean hadoopConf
+ spark.sessionState.catalog
checkpointDir = Utils.createDirectory(tempDir.getCanonicalPath,
"checkpoints").toString
sc.setCheckpointDir(checkpointDir)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index 464768a..1a30542 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -388,6 +388,8 @@ class SessionCatalog(
private def makeQualifiedTablePath(locationUri: URI, database: String): URI
= {
if (locationUri.isAbsolute) {
locationUri
+ } else if (new Path(locationUri).isAbsolute) {
+ makeQualifiedPath(locationUri)
} else {
val dbName = formatDatabaseName(database)
val dbLocation =
makeQualifiedDBPath(getDatabaseMetadata(dbName).locationUri)
diff --git
a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out
b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out
index ca1652b..ded27ab 100644
--- a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out
@@ -80,7 +80,7 @@ CREATE TABLE default.tbl (
b STRING,
c INT)
USING parquet
-LOCATION 'file:/path/to/table'
+LOCATION 'file:///path/to/table'
-- !query
@@ -110,7 +110,7 @@ CREATE TABLE default.tbl (
b STRING,
c INT)
USING parquet
-LOCATION 'file:/path/to/table'
+LOCATION 'file:///path/to/table'
-- !query
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index d9e3342..b64ed08 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -2773,7 +2773,7 @@ class DataSourceV2SQLSuite
val properties = table.properties
assert(properties.get(TableCatalog.PROP_PROVIDER) == "parquet")
assert(properties.get(TableCatalog.PROP_COMMENT) == "This is a comment")
- assert(properties.get(TableCatalog.PROP_LOCATION) == "file:/tmp")
+ assert(properties.get(TableCatalog.PROP_LOCATION) == "file:///tmp")
assert(properties.containsKey(TableCatalog.PROP_OWNER))
assert(properties.get(TableCatalog.PROP_EXTERNAL) == "true")
assert(properties.get(s"${TableCatalog.OPTION_PREFIX}from") == "0")
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala
index 1dd5e4a..ee8aa42 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala
@@ -53,7 +53,7 @@ trait ShowCreateTableSuiteBase extends
command.ShowCreateTableSuiteBase
|COMMENT 'This is a comment'
|TBLPROPERTIES ('prop1' = '1', 'prop2' = '2', 'prop3' = 3, 'prop4'
= 4)
|PARTITIONED BY (a)
- |LOCATION '/tmp'
+ |LOCATION 'file:/tmp'
""".stripMargin)
val showDDL = getShowCreateDDL(t)
assert(showDDL === Array(
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala
index 1aa8e37..bae793b 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala
@@ -29,7 +29,7 @@ import org.scalatest.BeforeAndAfter
import org.apache.spark.sql.AnalysisException
import
org.apache.spark.sql.catalyst.analysis.{NamespaceAlreadyExistsException,
NoSuchDatabaseException, NoSuchNamespaceException, NoSuchTableException,
TableAlreadyExistsException}
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
-import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier,
NamespaceChange, TableCatalog, TableChange, V1Table}
+import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier,
NamespaceChange, SupportsNamespaces, TableCatalog, TableChange, V1Table}
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types.{DoubleType, IntegerType, LongType,
StringType, StructField, StructType, TimestampType}
import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -60,7 +60,8 @@ class V2SessionCatalogTableSuite extends
V2SessionCatalogBaseSuite {
super.beforeAll()
val catalog = newCatalog()
catalog.createNamespace(Array("db"), emptyProps)
- catalog.createNamespace(Array("db2"), emptyProps)
+ catalog.createNamespace(Array("db2"),
+ Map(SupportsNamespaces.PROP_LOCATION -> "file:///db2.db").asJava)
catalog.createNamespace(Array("ns"), emptyProps)
catalog.createNamespace(Array("ns2"), emptyProps)
}
@@ -186,10 +187,17 @@ class V2SessionCatalogTableSuite extends
V2SessionCatalogBaseSuite {
assert(t2.catalogTable.location ===
makeQualifiedPathWithWarehouse("db.db/relative/path"))
catalog.dropTable(testIdent)
- // absolute path
+ // absolute path without scheme
properties.put(TableCatalog.PROP_LOCATION, "/absolute/path")
val t3 = catalog.createTable(testIdent, schema, Array.empty,
properties).asInstanceOf[V1Table]
- assert(t3.catalogTable.location.toString === "file:/absolute/path")
+ assert(t3.catalogTable.location.toString === "file:///absolute/path")
+ catalog.dropTable(testIdent)
+
+ // absolute path with scheme
+ properties.put(TableCatalog.PROP_LOCATION, "file:/absolute/path")
+ val t4 = catalog.createTable(testIdent, schema, Array.empty,
properties).asInstanceOf[V1Table]
+ assert(t4.catalogTable.location.toString === "file:/absolute/path")
+ catalog.dropTable(testIdent)
}
test("tableExists") {
@@ -685,10 +693,15 @@ class V2SessionCatalogTableSuite extends
V2SessionCatalogBaseSuite {
TableChange.setProperty(TableCatalog.PROP_LOCATION,
"relative/path")).asInstanceOf[V1Table]
assert(t2.catalogTable.location ===
makeQualifiedPathWithWarehouse("db.db/relative/path"))
- // absolute path
+ // absolute path without scheme
val t3 = catalog.alterTable(testIdent,
TableChange.setProperty(TableCatalog.PROP_LOCATION,
"/absolute/path")).asInstanceOf[V1Table]
- assert(t3.catalogTable.location.toString === "file:/absolute/path")
+ assert(t3.catalogTable.location.toString === "file:///absolute/path")
+
+ // absolute path with scheme
+ val t4 = catalog.alterTable(testIdent, TableChange.setProperty(
+ TableCatalog.PROP_LOCATION, "file:/absolute/path")).asInstanceOf[V1Table]
+ assert(t4.catalogTable.location.toString === "file:/absolute/path")
}
test("dropTable") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
