This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 8a8d1fb [SPARK-30234][SQL][FOLLOWUP] Rename
`spark.sql.legacy.addDirectory.recursive.enabled` to
`spark.sql.legacy.addSingleFileInAddFile`
8a8d1fb is described below
commit 8a8d1fbb10af6da481f26831cd519ef46ccbce6c
Author: iRakson <[email protected]>
AuthorDate: Sun Mar 1 10:55:41 2020 +0900
[SPARK-30234][SQL][FOLLOWUP] Rename
`spark.sql.legacy.addDirectory.recursive.enabled` to
`spark.sql.legacy.addSingleFileInAddFile`
### What changes were proposed in this pull request?
Rename `spark.sql.legacy.addDirectory.recursive.enabled` to
`spark.sql.legacy.addSingleFileInAddFile`
### Why are the changes needed?
To follow the naming convention
### Does this PR introduce any user-facing change?
No
### How was this patch tested?
Existing UTs.
Closes #27725 from iRakson/SPARK-30234_CONFIG.
Authored-by: iRakson <[email protected]>
Signed-off-by: HyukjinKwon <[email protected]>
(cherry picked from commit 92a5ae2ae488b5b0f106411d72e1c812b918ab93)
Signed-off-by: HyukjinKwon <[email protected]>
---
docs/sql-migration-guide.md | 2 +-
.../main/scala/org/apache/spark/sql/internal/SQLConf.scala | 12 ++++++------
.../org/apache/spark/sql/execution/command/resources.scala | 2 +-
.../org/apache/spark/sql/execution/command/DDLSuite.scala | 6 +++---
4 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index fb8f866..a9a39ce 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -326,7 +326,7 @@ license: |
</tr>
</table>
- - Since Spark 3.0, `ADD FILE` can be used to add file directories as well.
Earlier only single files can be added using this command. To restore the
behaviour of earlier versions, set
`spark.sql.legacy.addDirectory.recursive.enabled` to false.
+ - Since Spark 3.0, `ADD FILE` can be used to add file directories as well.
Earlier only single files can be added using this command. To restore the
behaviour of earlier versions, set `spark.sql.legacy.addSingleFileInAddFile` to
`true`.
- Since Spark 3.0, `SHOW TBLPROPERTIES` will cause `AnalysisException` if
the table does not exist. In Spark version 2.4 and earlier, this scenario
caused `NoSuchTableException`. Also, `SHOW TBLPROPERTIES` on a temporary view
will cause `AnalysisException`. In Spark version 2.4 and earlier, it returned
an empty result.
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 30f5fba..7718dd2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2152,13 +2152,13 @@ object SQLConf {
.booleanConf
.createWithDefault(false)
- val LEGACY_ADD_DIRECTORY_USING_RECURSIVE =
- buildConf("spark.sql.legacy.addDirectory.recursive.enabled")
+ val LEGACY_ADD_SINGLE_FILE_IN_ADD_FILE =
+ buildConf("spark.sql.legacy.addSingleFileInAddFile")
.internal()
- .doc("When true, users can add directory by passing path of a directory
to ADD FILE " +
- "command of SQL. If false, then only a single file can be added.")
+ .doc("When true, only a single file can be added using ADD FILE. If
false, then users " +
+ "can add directory by passing directory path to ADD FILE.")
.booleanConf
- .createWithDefault(true)
+ .createWithDefault(false)
val LEGACY_MSSQLSERVER_NUMERIC_MAPPING_ENABLED =
buildConf("spark.sql.legacy.mssqlserver.numericMapping.enabled")
@@ -2491,7 +2491,7 @@ class SQLConf extends Serializable with Logging {
def datetimeJava8ApiEnabled: Boolean = getConf(DATETIME_JAVA8API_ENABLED)
- def addDirectoryRecursiveEnabled: Boolean =
getConf(LEGACY_ADD_DIRECTORY_USING_RECURSIVE)
+ def addSingleFileInAddFile: Boolean =
getConf(LEGACY_ADD_SINGLE_FILE_IN_ADD_FILE)
def legacyMsSqlServerNumericMappingEnabled: Boolean =
getConf(LEGACY_MSSQLSERVER_NUMERIC_MAPPING_ENABLED)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/resources.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/resources.scala
index 1119e5c..549477d 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/resources.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/resources.scala
@@ -47,7 +47,7 @@ case class AddJarCommand(path: String) extends
RunnableCommand {
*/
case class AddFileCommand(path: String) extends RunnableCommand {
override def run(sparkSession: SparkSession): Seq[Row] = {
- val recursive = sparkSession.sessionState.conf.addDirectoryRecursiveEnabled
+ val recursive = !sparkSession.sessionState.conf.addSingleFileInAddFile
sparkSession.sparkContext.addFile(path, recursive)
Seq.empty[Row]
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index e4bf721..6c824c2 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -2982,16 +2982,16 @@ abstract class DDLSuite extends QueryTest with
SQLTestUtils {
}
}
- test("Add a directory when spark.sql.legacy.addDirectory.recursive.enabled
set to true") {
+ test(s"Add a directory when
${SQLConf.LEGACY_ADD_SINGLE_FILE_IN_ADD_FILE.key} set to false") {
val directoryToAdd = Utils.createTempDir("/tmp/spark/addDirectory/")
val testFile = File.createTempFile("testFile", "1", directoryToAdd)
spark.sql(s"ADD FILE $directoryToAdd")
assert(new
File(SparkFiles.get(s"${directoryToAdd.getName}/${testFile.getName}")).exists())
}
- test("Add a directory when spark.sql.legacy.addDirectory.recursive.enabled
not set to true") {
+ test(s"Add a directory when
${SQLConf.LEGACY_ADD_SINGLE_FILE_IN_ADD_FILE.key} set to true") {
withTempDir { testDir =>
- withSQLConf(SQLConf.LEGACY_ADD_DIRECTORY_USING_RECURSIVE.key -> "false")
{
+ withSQLConf(SQLConf.LEGACY_ADD_SINGLE_FILE_IN_ADD_FILE.key -> "true") {
val msg = intercept[SparkException] {
spark.sql(s"ADD FILE $testDir")
}.getMessage
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
