This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 7b379b77310c [SPARK-55127][SQL] Add avro_funcs group for Avro SQL
functions
7b379b77310c is described below
commit 7b379b77310c773e3f6eab2fc9a3a6b9496a8028
Author: Kent Yao <[email protected]>
AuthorDate: Fri Jan 23 07:47:42 2026 +0900
[SPARK-55127][SQL] Add avro_funcs group for Avro SQL functions
### What changes were proposed in this pull request?
This PR adds a new `avro_funcs` group for Avro SQL functions (from_avro,
to_avro, schema_of_avro) to follow the pattern used by other data format
functions like xml_funcs, json_funcs, and csv_funcs.
### Why are the changes needed?
Currently, Avro functions are grouped under `misc_funcs`, while other data
format functions have their own dedicated groups:
- `xml_funcs` for XML functions
- `json_funcs` for JSON functions
- `csv_funcs` for CSV functions
For consistency and better documentation organization, Avro functions
should also have their own group.
### Does this PR introduce _any_ user-facing change?
No, this is a documentation-only change that affects how functions are
grouped in the SQL function reference.
### How was this patch tested?
Existing tests pass. The change was validated by building the documentation.
### Was this patch authored or co-authored using generative AI tooling?
Yes, GitHub Copilot was used to assist with this change.
Closes #53909 from yaooqinn/SPARK-55127-avro-funcs-group.
Authored-by: Kent Yao <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../spark/sql/catalyst/expressions/ExpressionInfo.java | 12 ++++++------
.../spark/sql/catalyst/expressions/avroSqlFunctions.scala | 6 +++---
.../apache/spark/sql/expressions/ExpressionInfoSuite.scala | 2 +-
3 files changed, 10 insertions(+), 10 deletions(-)
diff --git
a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java
b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java
index 592230cc5b10..ff4e2dfa0d4f 100644
---
a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java
+++
b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java
@@ -44,12 +44,12 @@ public class ExpressionInfo {
private String source;
private static final Set<String> validGroups =
- new HashSet<>(Arrays.asList("agg_funcs", "array_funcs",
"binary_funcs", "bitwise_funcs",
- "collection_funcs", "predicate_funcs", "conditional_funcs",
"conversion_funcs",
- "csv_funcs", "datetime_funcs", "generator_funcs", "hash_funcs",
"json_funcs",
- "lambda_funcs", "map_funcs", "math_funcs", "misc_funcs",
"string_funcs", "struct_funcs",
- "window_funcs", "xml_funcs", "table_funcs", "url_funcs",
"variant_funcs",
- "vector_funcs", "st_funcs"));
+ new HashSet<>(Arrays.asList("agg_funcs", "array_funcs", "avro_funcs",
"binary_funcs",
+ "bitwise_funcs", "collection_funcs", "predicate_funcs",
"conditional_funcs",
+ "conversion_funcs", "csv_funcs", "datetime_funcs",
"generator_funcs", "hash_funcs",
+ "json_funcs", "lambda_funcs", "map_funcs", "math_funcs",
"misc_funcs",
+ "string_funcs", "struct_funcs", "window_funcs", "xml_funcs",
"table_funcs",
+ "url_funcs", "variant_funcs", "vector_funcs", "st_funcs"));
private static final Set<String> validSources =
new HashSet<>(Arrays.asList("built-in", "hive", "python_udf",
"scala_udf", "sql_udf",
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/avroSqlFunctions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/avroSqlFunctions.scala
index 6693ee83fd4a..aea04913e468 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/avroSqlFunctions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/avroSqlFunctions.scala
@@ -51,7 +51,7 @@ import org.apache.spark.util.Utils
To deserialize the data with a compatible and evolved schema, the expected
Avro schema can be
set via the corresponding option.
""",
- group = "misc_funcs",
+ group = "avro_funcs",
since = "4.0.0"
)
// scalastyle:on line.size.limit
@@ -148,7 +148,7 @@ case class FromAvro(child: Expression, jsonFormatSchema:
Expression, options: Ex
> SELECT _FUNC_(s) IS NULL FROM (SELECT NULL AS s);
[true]
""",
- group = "misc_funcs",
+ group = "avro_funcs",
since = "4.0.0"
)
// scalastyle:on line.size.limit
@@ -220,7 +220,7 @@ case class ToAvro(child: Expression, jsonFormatSchema:
Expression)
> SELECT _FUNC_('{"type": "record", "name": "struct", "fields":
[{"name": "u", "type": ["int", "string"]}]}', map());
STRUCT<u: STRUCT<member0: INT, member1: STRING> NOT NULL>
""",
- group = "misc_funcs",
+ group = "avro_funcs",
since = "4.0.0"
)
// scalastyle:on line.size.limit
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
index 2b1d4e7e7df4..dc2942572ed7 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
@@ -56,7 +56,7 @@ class ExpressionInfoSuite extends SparkFunSuite with
SharedSparkSession {
}
val validGroups = Seq(
- "agg_funcs", "array_funcs", "binary_funcs", "bitwise_funcs",
"collection_funcs",
+ "agg_funcs", "array_funcs", "avro_funcs", "binary_funcs",
"bitwise_funcs", "collection_funcs",
"predicate_funcs", "conditional_funcs", "conversion_funcs", "csv_funcs",
"datetime_funcs",
"generator_funcs", "hash_funcs", "json_funcs", "lambda_funcs",
"map_funcs", "math_funcs",
"misc_funcs", "string_funcs", "struct_funcs", "window_funcs",
"xml_funcs", "table_funcs",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
