This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a8ad71dbe417 [SPARK-46998][SQL] Deprecate the SQL config
`spark.sql.legacy.allowZeroIndexInFormatString`
a8ad71dbe417 is described below
commit a8ad71dbe417c16af13d46783e13fba0c2280268
Author: Max Gekk <[email protected]>
AuthorDate: Wed Feb 7 18:20:20 2024 -0800
[SPARK-46998][SQL] Deprecate the SQL config
`spark.sql.legacy.allowZeroIndexInFormatString`
### What changes were proposed in this pull request?
In the PR, I propose to deprecate the SQL config
`spark.sql.legacy.allowZeroIndexInFormatString` and put it to the list
`deprecatedSQLConfigs` in `SQLConf`.
### Why are the changes needed?
After migration on JDK 17+, the SQL config
`spark.sql.legacy.allowZeroIndexInFormatString` doesn't work anymore, and
doesn't allow to use the zero index. Even users set the config to `true`, they
get the error:
```java
Illegal format argument index = 0
java.util.IllegalFormatArgumentIndexException: Illegal format argument
index = 0
at
java.base/java.util.Formatter$FormatSpecifier.index(Formatter.java:2808)
at
java.base/java.util.Formatter$FormatSpecifier.<init>(Formatter.java:2879)
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
By running the modified test suite:
```
$ build/sbt "test:testOnly *QueryCompilationErrorsSuite"
```
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #45057 from MaxGekk/deprecate-ALLOW_ZERO_INDEX_IN_FORMAT_STRING.
Authored-by: Max Gekk <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
docs/sql-migration-guide.md | 1 +
.../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala | 7 +++++--
.../org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala | 7 +++++++
3 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index cb5e697f871c..3d0c7280496a 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -38,6 +38,7 @@ license: |
- `spark.sql.avro.datetimeRebaseModeInRead` instead of
`spark.sql.legacy.avro.datetimeRebaseModeInRead`
- Since Spark 4.0, the default value of `spark.sql.orc.compression.codec` is
changed from `snappy` to `zstd`. To restore the previous behavior, set
`spark.sql.orc.compression.codec` to `snappy`.
- Since Spark 4.0, `SELECT (*)` is equivalent to `SELECT struct(*)` instead of
`SELECT *`. To restore the previous behavior, set
`spark.sql.legacy.ignoreParenthesesAroundStar` to `true`.
+- Since Spark 4.0, the SQL config
`spark.sql.legacy.allowZeroIndexInFormatString` is deprecated. Consider to
change `strfmt` of the `format_string` function to use 1-based indexes. The
first argument must be referenced by "1$", the second by "2$", etc.
## Upgrading from Spark SQL 3.4 to 3.5
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 8f86a1c8a1f3..59db3e71a135 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2339,7 +2339,8 @@ object SQLConf {
.doc("When false, the `strfmt` in `format_string(strfmt, obj, ...)` and
" +
"`printf(strfmt, obj, ...)` will no longer support to use \"0$\" to
specify the first " +
"argument, the first argument should always reference by \"1$\" when
use argument index " +
- "to indicating the position of the argument in the argument list.")
+ "to indicating the position of the argument in the argument list. " +
+ "This config will be removed in the future releases.")
.version("3.3")
.booleanConf
.createWithDefault(false)
@@ -4718,7 +4719,9 @@ object SQLConf {
DeprecatedConfig(ESCAPED_STRING_LITERALS.key, "4.0",
"Use raw string literals with the `r` prefix instead. "),
DeprecatedConfig("spark.connect.copyFromLocalToFs.allowDestLocal", "4.0",
- s"Use '${ARTIFACT_COPY_FROM_LOCAL_TO_FS_ALLOW_DEST_LOCAL.key}'
instead.")
+ s"Use '${ARTIFACT_COPY_FROM_LOCAL_TO_FS_ALLOW_DEST_LOCAL.key}'
instead."),
+ DeprecatedConfig(ALLOW_ZERO_INDEX_IN_FORMAT_STRING.key, "4.0", "Increase
indexes by 1 " +
+ "in `strfmt` of the `format_string` function. Refer to the first
argument by \"1$\".")
)
Map(configs.map { cfg => cfg.key -> cfg } : _*)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
index 7ea0c3843444..e22399c326f6 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.errors
+import java.util.IllegalFormatException
+
import org.apache.spark.{SPARK_DOC_ROOT, SparkIllegalArgumentException,
SparkUnsupportedOperationException}
import org.apache.spark.sql._
import org.apache.spark.sql.api.java.{UDF1, UDF2, UDF23Test}
@@ -129,6 +131,11 @@ class QueryCompilationErrorsSuite
context = ExpectedContext(
fragment = "format_string('%0$s', 'Hello')", start = 7, stop = 36))
}
+ withSQLConf(SQLConf.ALLOW_ZERO_INDEX_IN_FORMAT_STRING.key -> "true") {
+ intercept[IllegalFormatException] {
+ sql("select format_string('%0$s', 'Hello')").collect()
+ }
+ }
}
test("INVALID_PANDAS_UDF_PLACEMENT: Using aggregate function with grouped
aggregate pandas UDF") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
