ganeshashree commented on code in PR #58005: URL: https://github.com/apache/spark/pull/58005#discussion_r3854372503
########## sql/core/src/test/scala/org/apache/spark/sql/JsonArraySuite.scala: ########## @@ -0,0 +1,644 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.SparkRuntimeException +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch +import org.apache.spark.sql.catalyst.expressions.{Cast, Collate, JsonArray, JsonConstructorNullBehavior, JsonQuery, JsonQueryBehavior, JsonQueryQuotes, JsonQueryWrapper, Literal, ResolvedCollation} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.{CharType, GeographyType, GeometryType, IntegerType, MapType, StringType, VarcharType} + +/** + * Test suite for the `JSON_ARRAY` ANSI SQL:2016 constructor function. + */ +class JsonArraySuite extends QueryTest with SharedSparkSession { + + import testImplicits._ + + test("JSON_ARRAY with simple scalar values") { + checkAnswer( + sql("SELECT JSON_ARRAY(1, 'x', true)"), + Row("""[1,"x",true]""")) + } + + test("JSON_ARRAY with NULL elements - ABSENT ON NULL (default)") { + checkAnswer( + sql("SELECT JSON_ARRAY(1, NULL, 3)"), + Row("[1,3]")) + } + + test("JSON_ARRAY with NULL elements - NULL ON NULL") { + checkAnswer( + sql("SELECT JSON_ARRAY(1, NULL, 3 NULL ON NULL)"), + Row("[1,null,3]")) + } + + test("JSON_ARRAY with NULL elements - explicit ABSENT ON NULL") { + // Exercise the explicit `ABSENT ON NULL` grammar branch (the default is implicit absent, so + // this spelling is otherwise untested); it drops NULL elements just like the default. + checkAnswer( + sql("SELECT JSON_ARRAY(1, NULL, 3 ABSENT ON NULL)"), + Row("[1,3]")) + checkAnswer( + sql("SELECT JSON_ARRAY(1, NULL, 3 ABSENT ON NULL RETURNING STRING)"), + Row("[1,3]")) + } + + test("JSON_ARRAY with empty list") { + checkAnswer( + sql("SELECT JSON_ARRAY()"), + Row("[]")) + } + + test("JSON_ARRAY with floating point numbers") { + checkAnswer( + sql("SELECT JSON_ARRAY(1.5, 2.7)"), + Row("[1.5,2.7]")) + } + + test("JSON_ARRAY with mixed types") { + checkAnswer( + sql("SELECT JSON_ARRAY(1, 'text', 3.14, true, false)"), + Row("""[1,"text",3.14,true,false]""")) + } + + test("JSON_ARRAY with all NULLs and ABSENT ON NULL") { + checkAnswer( + sql("SELECT JSON_ARRAY(NULL, NULL)"), + Row("[]")) + } + + test("JSON_ARRAY with RETURNING STRING (explicit)") { + checkAnswer( + sql("SELECT JSON_ARRAY(1, 2, 3 RETURNING STRING)"), + Row("[1,2,3]")) + } + + test("JSON_ARRAY with both NULL ON NULL and RETURNING clauses") { + // The grammar allows `... ON NULL` and `RETURNING` together, in that order; exercise both. + checkAnswer( + sql("SELECT JSON_ARRAY(1, NULL, 3 NULL ON NULL RETURNING STRING)"), + Row("[1,null,3]")) + } + + test("JSON_ARRAY over non-foldable columns exercises row-wise eval") { + val df = Seq((1, "a", true), (2, "b", false)).toDF("i", "s", "b") + checkAnswer( + df.selectExpr("JSON_ARRAY(i, s, b)"), + Seq(Row("""[1,"a",true]"""), Row("""[2,"b",false]"""))) + } + + test("JSON_ARRAY renders decimals and dates via Jackson, not toString") { + checkAnswer( + sql("SELECT JSON_ARRAY(CAST(1.50 AS DECIMAL(5,2)), DATE'2020-01-02')"), + Row("""[1.50,"2020-01-02"]""")) + } + + test("JSON_ARRAY renders a TIMESTAMP via to_json's writer in the session time zone") { + // The constructor is TimeZoneAware and shares to_json's writer, so a TIMESTAMP element must + // render identically to to_json of the singleton array, formatted in the session time zone. + // Assert agreement with that writer (rather than pinning a fragile format string), and that the + // rendering tracks the session time zone by differing between two zones. + def render(tz: String): String = withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz) { + val out = + sql("SELECT JSON_ARRAY(TIMESTAMP'2020-01-02 03:04:05')").collect().head.getString(0) + val expected = + sql("SELECT to_json(array(TIMESTAMP'2020-01-02 03:04:05'))").collect().head.getString(0) + assert(out == expected, s"for tz=$tz") + out + } + assert(render("UTC") != render("America/Los_Angeles")) + } + + test("JSON_ARRAY renders array and map elements as JSON structures, like to_json") { + // The docs state array/map/struct arguments render via the same writer as to_json (as nested + // JSON structures, not quoted strings). Cover arrays and maps explicitly (structs are covered + // by the ignoreNullFields test); a nested array element serializes to [1,2], a map to {"k":1}. + checkAnswer( + sql("SELECT JSON_ARRAY(array(1, 2), map('k', 1))"), + Row("""[[1,2],{"k":1}]""")) + checkAnswer( + sql("SELECT JSON_ARRAY(array(array(1), array(2, 3)))"), + Row("[[[1],[2,3]]]")) + } + + test("JSON_ARRAY strings are escaped") { + checkAnswer( + sql("""SELECT JSON_ARRAY('a"b', 'c\td')"""), + Row("""["a\"b","c\td"]""")) + } + + test("nested JSON_ARRAY is spliced raw, not re-quoted (implicit FORMAT JSON)") { + checkAnswer( + sql("SELECT JSON_ARRAY(JSON_ARRAY(1, 2), 3)"), + Row("[[1,2],3]")) + checkAnswer( + sql("SELECT JSON_ARRAY(JSON_ARRAY(1))"), + Row("[[1]]")) + } + + test("explicit FORMAT JSON splices a string verbatim; a plain string is quoted") { + // A plain string element is quoted and escaped like any other string value... + checkAnswer(sql("""SELECT JSON_ARRAY('[1,2]')"""), Row("""["[1,2]"]""")) + // ...while FORMAT JSON marks it as already-JSON text, spliced in verbatim. + checkAnswer(sql("""SELECT JSON_ARRAY('[1,2]' FORMAT JSON)"""), Row("[[1,2]]")) + checkAnswer( + sql("""SELECT JSON_ARRAY('{"a":1}' FORMAT JSON, 'x')"""), + Row("""[{"a":1},"x"]""")) + } + + test("splicing is decided from the source, not the optimized plan shape") { + // A JSON_ARRAY result surfaced as a column is a plain STRING and must be quoted -- even though + // CollapseProject may inline the inner JSON_ARRAY into the outer argument position. The FORMAT + // JSON decision is frozen from the lexical argument at parse time, so it does not depend on + // whether that inlining happens: the result is ["[1]"], never [[1]]. + val inlined = sql("SELECT JSON_ARRAY(a) AS r FROM (SELECT JSON_ARRAY(1) AS a) t") + checkAnswer(inlined, Row("""["[1]"]""")) + // Referencing the alias twice blocks CollapseProject from inlining it; the result is identical, Review Comment: Done. Switched the producer to a non-foldable `JSON_ARRAY(id)` over range(1) so the two cases have genuinely different shapes, and added `optimizedPlan` assertions (`projectCount == 1` when the single-ref alias is inlined, `== 2` when the twice-referenced non-foldable alias blocks `CollapseProject`). Both still yield the quoted `["[0]"]`, confirming the splice decision is independent of plan shape. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
