ganeshashree commented on code in PR #58034:
URL: https://github.com/apache/spark/pull/58034#discussion_r4282658718
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/json/JsonExpressionEvalUtils.scala:
##########
@@ -218,6 +218,123 @@ case class StructsToJsonEvaluator(
}
}
+/**
+ * Renders a JSON_OBJECT value by writing every member into one
[[JacksonGenerator]] lifecycle,
+ * rather than serializing each member separately and splicing the fragments.
Shared by the
+ * interpreted and codegen paths of `JsonObjectExpr`.
+ *
+ * `evaluate` takes a carrier row of the already-evaluated members flattened as
+ * `[key0, value0, key1, value1, ...]`. Keys are non-null strings: the caller
guards them so a null
+ * key raises JSON_OBJECT_NULL_KEY before its value is evaluated. Per member
it writes the field
+ * name then the value -- a raw (`FORMAT JSON`) value spliced verbatim
(validated first when it is
+ * arbitrary user text), a null written as JSON `null` or dropped under ABSENT
ON NULL, any other
+ * value rendered through the shared generator's type writer.
+ */
+case class JsonObjectEvaluator(
+ valueTypes: Seq[DataType],
+ rawJson: Seq[Boolean],
+ needsValidation: Seq[Boolean],
+ valueFoldable: Seq[Boolean],
+ keepNullMembers: Boolean,
+ timeZoneId: String,
+ funcName: String) {
+
+ @transient private lazy val writer = new CharArrayWriter()
+
+ // Schema the generator is initialized with: alternating non-null string
keys and the members'
+ // value types. Only the per-type value writers and the member-write
primitives are used; the
+ // whole-row `write` path is never taken.
+ @transient private lazy val carrierSchema: StructType = {
+ val fields = Array.newBuilder[StructField]
+ var i = 0
+ while (i < valueTypes.length) {
+ fields += StructField(s"k$i", StringType)
+ fields += StructField(s"v$i", valueTypes(i))
+ i += 1
+ }
+ StructType(fields.result())
+ }
+
+ @transient private lazy val gen =
+ new JacksonGenerator(carrierSchema, writer, new JSONOptions(Map.empty,
timeZoneId))
+
+ @transient private lazy val valueWriters: Array[(SpecializedGetters, Int) =>
Unit] =
+ valueTypes.map(gen.makeValueWriter).toArray
+
+ @transient private lazy val formatJsonFactory = new JsonFactory()
+
+ @transient private lazy val cachedValidatedTexts: Array[String] =
+ new Array[String](valueTypes.length)
+
+ // Raw text to splice for member `idx`: a trusted nested constructor as-is;
an explicit
+ // `FORMAT JSON` string validated once (cached) when foldable, else per row.
+ private def rawText(idx: Int, value: UTF8String): String = {
+ if (!needsValidation(idx)) {
+ value.toString
+ } else if (valueFoldable(idx)) {
+ val cached = cachedValidatedTexts(idx)
+ if (cached != null) {
+ cached
+ } else {
+ val text = value.toString
+ validate(idx, text)
+ cachedValidatedTexts(idx) = text
+ text
+ }
+ } else {
+ val text = value.toString
+ validate(idx, text)
+ text
+ }
+ }
+
+ // A spliced value must be exactly one well-formed JSON value: parse one,
reject trailing text.
+ private def validate(idx: Int, text: String): Unit = {
+ val valid =
+ try {
+ Utils.tryWithResource(formatJsonFactory.createParser(text)) { parser =>
+ if (parser.nextToken() == null) {
+ false
+ } else {
+ parser.skipChildren()
+ parser.nextToken() == null
+ }
+ }
+ } catch {
+ case _: JsonProcessingException => false
+ }
+ if (!valid) {
+ // Flattened 1-based value-arg position (member `idx` is arg `2*idx+2`).
+ throw QueryExecutionErrors.invalidJsonFormatJsonValueError(funcName, idx
* 2 + 2, text)
+ }
+ }
+
+ final def evaluate(row: InternalRow): Any = {
+ gen.writeStartObject()
+ var i = 0
+ while (i < valueTypes.length) {
+ val valOrd = i * 2 + 1
+ val isNull = row.isNullAt(valOrd)
+ if (!isNull || keepNullMembers) {
+ gen.writeFieldName(row.getUTF8String(i * 2).toString)
+ if (isNull) {
+ gen.writeNull()
+ } else if (rawJson(i)) {
+ gen.writeRawValue(rawText(i, row.getUTF8String(valOrd)))
Review Comment:
Done — `evaluate` now resolves and validates every FORMAT JSON value in a
pre-pass before `writeStartObject`, so a caught error leaves the generator at
root and the writer empty. Added a `try_to_binary` regression over a
malformed-then-valid row sequence under both interpreted and whole-stage
codegen.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/json/JsonExpressionEvalUtils.scala:
##########
@@ -218,6 +218,123 @@ case class StructsToJsonEvaluator(
}
}
+/**
+ * Renders a JSON_OBJECT value by writing every member into one
[[JacksonGenerator]] lifecycle,
+ * rather than serializing each member separately and splicing the fragments.
Shared by the
+ * interpreted and codegen paths of `JsonObjectExpr`.
+ *
+ * `evaluate` takes a carrier row of the already-evaluated members flattened as
+ * `[key0, value0, key1, value1, ...]`. Keys are non-null strings: the caller
guards them so a null
+ * key raises JSON_OBJECT_NULL_KEY before its value is evaluated. Per member
it writes the field
+ * name then the value -- a raw (`FORMAT JSON`) value spliced verbatim
(validated first when it is
+ * arbitrary user text), a null written as JSON `null` or dropped under ABSENT
ON NULL, any other
+ * value rendered through the shared generator's type writer.
+ */
+case class JsonObjectEvaluator(
+ valueTypes: Seq[DataType],
+ rawJson: Seq[Boolean],
+ needsValidation: Seq[Boolean],
+ valueFoldable: Seq[Boolean],
+ keepNullMembers: Boolean,
+ timeZoneId: String,
+ funcName: String) {
+
+ @transient private lazy val writer = new CharArrayWriter()
+
+ // Schema the generator is initialized with: alternating non-null string
keys and the members'
+ // value types. Only the per-type value writers and the member-write
primitives are used; the
+ // whole-row `write` path is never taken.
+ @transient private lazy val carrierSchema: StructType = {
+ val fields = Array.newBuilder[StructField]
+ var i = 0
+ while (i < valueTypes.length) {
+ fields += StructField(s"k$i", StringType)
+ fields += StructField(s"v$i", valueTypes(i))
+ i += 1
+ }
+ StructType(fields.result())
+ }
+
+ @transient private lazy val gen =
+ new JacksonGenerator(carrierSchema, writer, new JSONOptions(Map.empty,
timeZoneId))
+
+ @transient private lazy val valueWriters: Array[(SpecializedGetters, Int) =>
Unit] =
+ valueTypes.map(gen.makeValueWriter).toArray
+
+ @transient private lazy val formatJsonFactory = new JsonFactory()
+
+ @transient private lazy val cachedValidatedTexts: Array[String] =
+ new Array[String](valueTypes.length)
+
+ // Raw text to splice for member `idx`: a trusted nested constructor as-is;
an explicit
+ // `FORMAT JSON` string validated once (cached) when foldable, else per row.
+ private def rawText(idx: Int, value: UTF8String): String = {
+ if (!needsValidation(idx)) {
+ value.toString
+ } else if (valueFoldable(idx)) {
+ val cached = cachedValidatedTexts(idx)
+ if (cached != null) {
+ cached
+ } else {
+ val text = value.toString
+ validate(idx, text)
+ cachedValidatedTexts(idx) = text
+ text
+ }
+ } else {
+ val text = value.toString
+ validate(idx, text)
+ text
+ }
+ }
+
+ // A spliced value must be exactly one well-formed JSON value: parse one,
reject trailing text.
+ private def validate(idx: Int, text: String): Unit = {
+ val valid =
+ try {
+ Utils.tryWithResource(formatJsonFactory.createParser(text)) { parser =>
+ if (parser.nextToken() == null) {
+ false
+ } else {
+ parser.skipChildren()
+ parser.nextToken() == null
+ }
+ }
+ } catch {
+ case _: JsonProcessingException => false
+ }
+ if (!valid) {
+ // Flattened 1-based value-arg position (member `idx` is arg `2*idx+2`).
+ throw QueryExecutionErrors.invalidJsonFormatJsonValueError(funcName, idx
* 2 + 2, text)
+ }
+ }
+
+ final def evaluate(row: InternalRow): Any = {
+ gen.writeStartObject()
+ var i = 0
+ while (i < valueTypes.length) {
+ val valOrd = i * 2 + 1
+ val isNull = row.isNullAt(valOrd)
+ if (!isNull || keepNullMembers) {
+ gen.writeFieldName(row.getUTF8String(i * 2).toString)
+ if (isNull) {
+ gen.writeNull()
+ } else if (rawJson(i)) {
Review Comment:
Done — the evaluator now backs the per-row flags with arrays and caches the
member count, so the row loop stays linear regardless of the `Seq` the caller
passes.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/json/JsonExpressionEvalUtils.scala:
##########
@@ -218,6 +218,123 @@ case class StructsToJsonEvaluator(
}
}
+/**
+ * Renders a JSON_OBJECT value by writing every member into one
[[JacksonGenerator]] lifecycle,
+ * rather than serializing each member separately and splicing the fragments.
Shared by the
+ * interpreted and codegen paths of `JsonObjectExpr`.
+ *
+ * `evaluate` takes a carrier row of the already-evaluated members flattened as
+ * `[key0, value0, key1, value1, ...]`. Keys are non-null strings: the caller
guards them so a null
+ * key raises JSON_OBJECT_NULL_KEY before its value is evaluated. Per member
it writes the field
+ * name then the value -- a raw (`FORMAT JSON`) value spliced verbatim
(validated first when it is
+ * arbitrary user text), a null written as JSON `null` or dropped under ABSENT
ON NULL, any other
+ * value rendered through the shared generator's type writer.
+ */
+case class JsonObjectEvaluator(
+ valueTypes: Seq[DataType],
+ rawJson: Seq[Boolean],
+ needsValidation: Seq[Boolean],
+ valueFoldable: Seq[Boolean],
+ keepNullMembers: Boolean,
+ timeZoneId: String,
+ funcName: String) {
+
+ @transient private lazy val writer = new CharArrayWriter()
+
+ // Schema the generator is initialized with: alternating non-null string
keys and the members'
+ // value types. Only the per-type value writers and the member-write
primitives are used; the
+ // whole-row `write` path is never taken.
+ @transient private lazy val carrierSchema: StructType = {
+ val fields = Array.newBuilder[StructField]
+ var i = 0
+ while (i < valueTypes.length) {
+ fields += StructField(s"k$i", StringType)
+ fields += StructField(s"v$i", valueTypes(i))
+ i += 1
+ }
+ StructType(fields.result())
+ }
+
+ @transient private lazy val gen =
+ new JacksonGenerator(carrierSchema, writer, new JSONOptions(Map.empty,
timeZoneId))
+
+ @transient private lazy val valueWriters: Array[(SpecializedGetters, Int) =>
Unit] =
+ valueTypes.map(gen.makeValueWriter).toArray
+
+ @transient private lazy val formatJsonFactory = new JsonFactory()
+
+ @transient private lazy val cachedValidatedTexts: Array[String] =
+ new Array[String](valueTypes.length)
+
+ // Raw text to splice for member `idx`: a trusted nested constructor as-is;
an explicit
+ // `FORMAT JSON` string validated once (cached) when foldable, else per row.
+ private def rawText(idx: Int, value: UTF8String): String = {
+ if (!needsValidation(idx)) {
+ value.toString
+ } else if (valueFoldable(idx)) {
+ val cached = cachedValidatedTexts(idx)
+ if (cached != null) {
Review Comment:
Done — added a multi-row test with two constant FORMAT JSON members at
different positions plus a column-valued member (so the object isn't
constant-folded); a wrong fill or read index would splice the other member's
text.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala:
##########
@@ -2273,3 +2332,337 @@ case class JsonTypeof(child: Expression)
override protected def withNewChildInternal(newChild: Expression):
JsonTypeof =
copy(child = newChild)
}
+
+/**
+ * The SQL:2016 `JSON_OBJECT` constructor function (feature T811): constructs
a JSON object from
+ * key-value pairs, written `key VALUE value`, `KEY key VALUE value`, `key :
value`, or the
+ * MySQL-style `key, value`.
+ *
+ * Keys must be non-null strings; a null key is an error. The `ON NULL` clause
controls whether
+ * null-valued pairs are included (NULL ON NULL, the standard default) or
omitted (ABSENT ON NULL).
+ *
+ * A value is spliced in raw (unquoted) when it carries `FORMAT JSON` (tracked
per member by
+ * `rawJson`): the explicit clause (e.g. `JSON_OBJECT('a' VALUE '{"b":1}'
FORMAT JSON)`), or the
+ * implicit form carried by a nested JSON constructor / `JSON_QUERY` under
KEEP QUOTES (see
+ * `rawJsonValue`). Explicit `FORMAT JSON` on a non-constructor is arbitrary
user text, so
+ * `needsValidation` marks it for JSON validation at eval; a nested
constructor is trusted.
+ *
+ * Examples:
+ * JSON_OBJECT('id' VALUE 7, 'name' VALUE 'Ada') ->
'{"id":7,"name":"Ada"}'
+ * JSON_OBJECT('id': 7, 'v': NULL) -> '{"id":7,"v":null}'
+ * JSON_OBJECT('id': 7, 'v': NULL ABSENT ON NULL) -> '{"id":7}'
+ * JSON_OBJECT() -> '{}'
+ *
+ * A clause-free call that is not itself lexically nested in an enclosing
constructor's *value*
+ * position routes through function resolution and is rebuilt by
`JsonObjectExpressionBuilder`, so a
+ * same-named routine can shadow the built-in `json_object`: always for the
comma form (a nested
+ * producer value routes too, carrying its raw-splice eligibility via
`JsonImplicitFormatCarrier`),
+ * but for the `VALUE` / `:` forms only when no member carries a raw value
(`rawJson.contains(true)`
+ * -- an explicit `FORMAT JSON` or a nested producer in *value* position --
keeps it direct).
+ * Otherwise it is built directly from the grammar (clause-bearing, or
value-nested); a nested
+ * *key*-position call stays routable (see `AstBuilder.visitJsonObject`).
+ * The user-facing reference lives in
`docs/sql-ref-syntax-qry-select-json-object.md`.
+ */
+/**
+ * Guards a JSON_OBJECT key: returns the key, or raises JSON_OBJECT_NULL_KEY
when it is null. Wraps
+ * each key inside `JsonObjectExpr.replacement` so that, as the carrier struct
is evaluated left to
+ * right, a null key fails before its member's value is evaluated (the
key-before-value error
+ * order). Internal, so it is `NonSQLExpression`; it never appears in
`JsonObjectExpr.sql`.
+ */
+case class JsonObjectKeyGuard(child: Expression) extends UnaryExpression with
NonSQLExpression {
+ override def dataType: DataType = child.dataType
+ // Can throw, so it is not foldable (mirrors AssertNotNull).
+ override def foldable: Boolean = false
+ // Raises on a null key, so the result is never null.
+ override def nullable: Boolean = false
+
+ override def eval(input: InternalRow): Any = {
+ val value = child.eval(input)
+ if (value == null) {
+ throw QueryExecutionErrors.jsonObjectNullKeyError()
+ }
+ value
+ }
+
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode = {
+ val childGen = child.genCode(ctx)
+ val code = childGen.code + code"""
+ if (${childGen.isNull}) {
+ throw QueryExecutionErrors.jsonObjectNullKeyError();
+ }
+ """
+ ev.copy(code = code, isNull = FalseLiteral, value = childGen.value)
+ }
+
+ override protected def withNewChildInternal(newChild: Expression):
JsonObjectKeyGuard =
+ copy(child = newChild)
+}
+
+case class JsonObjectExpr(
+ members: Seq[(Expression, Expression)],
+ rawJson: Seq[Boolean],
+ needsValidation: Seq[Boolean],
+ nullBehavior: JsonConstructorNullBehavior =
JsonConstructorNullBehavior.Null,
Review Comment:
Done — the type and case docs now cover both constructors and note the
differing defaults (ABSENT ON NULL for JSON_ARRAY, NULL ON NULL for
JSON_OBJECT).
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala:
##########
@@ -2273,3 +2332,337 @@ case class JsonTypeof(child: Expression)
override protected def withNewChildInternal(newChild: Expression):
JsonTypeof =
copy(child = newChild)
}
+
+/**
+ * The SQL:2016 `JSON_OBJECT` constructor function (feature T811): constructs
a JSON object from
+ * key-value pairs, written `key VALUE value`, `KEY key VALUE value`, `key :
value`, or the
+ * MySQL-style `key, value`.
+ *
+ * Keys must be non-null strings; a null key is an error. The `ON NULL` clause
controls whether
+ * null-valued pairs are included (NULL ON NULL, the standard default) or
omitted (ABSENT ON NULL).
+ *
+ * A value is spliced in raw (unquoted) when it carries `FORMAT JSON` (tracked
per member by
+ * `rawJson`): the explicit clause (e.g. `JSON_OBJECT('a' VALUE '{"b":1}'
FORMAT JSON)`), or the
+ * implicit form carried by a nested JSON constructor / `JSON_QUERY` under
KEEP QUOTES (see
+ * `rawJsonValue`). Explicit `FORMAT JSON` on a non-constructor is arbitrary
user text, so
+ * `needsValidation` marks it for JSON validation at eval; a nested
constructor is trusted.
+ *
+ * Examples:
+ * JSON_OBJECT('id' VALUE 7, 'name' VALUE 'Ada') ->
'{"id":7,"name":"Ada"}'
+ * JSON_OBJECT('id': 7, 'v': NULL) -> '{"id":7,"v":null}'
+ * JSON_OBJECT('id': 7, 'v': NULL ABSENT ON NULL) -> '{"id":7}'
+ * JSON_OBJECT() -> '{}'
+ *
+ * A clause-free call that is not itself lexically nested in an enclosing
constructor's *value*
+ * position routes through function resolution and is rebuilt by
`JsonObjectExpressionBuilder`, so a
+ * same-named routine can shadow the built-in `json_object`: always for the
comma form (a nested
+ * producer value routes too, carrying its raw-splice eligibility via
`JsonImplicitFormatCarrier`),
+ * but for the `VALUE` / `:` forms only when no member carries a raw value
(`rawJson.contains(true)`
+ * -- an explicit `FORMAT JSON` or a nested producer in *value* position --
keeps it direct).
+ * Otherwise it is built directly from the grammar (clause-bearing, or
value-nested); a nested
+ * *key*-position call stays routable (see `AstBuilder.visitJsonObject`).
+ * The user-facing reference lives in
`docs/sql-ref-syntax-qry-select-json-object.md`.
+ */
+/**
Review Comment:
Done — moved the JSON_OBJECT Scaladoc directly above `JsonObjectExpr`, with
`JsonObjectKeyGuard` and its own doc now preceding it.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala:
##########
@@ -2273,3 +2332,337 @@ case class JsonTypeof(child: Expression)
override protected def withNewChildInternal(newChild: Expression):
JsonTypeof =
copy(child = newChild)
}
+
+/**
+ * The SQL:2016 `JSON_OBJECT` constructor function (feature T811): constructs
a JSON object from
+ * key-value pairs, written `key VALUE value`, `KEY key VALUE value`, `key :
value`, or the
+ * MySQL-style `key, value`.
+ *
+ * Keys must be non-null strings; a null key is an error. The `ON NULL` clause
controls whether
+ * null-valued pairs are included (NULL ON NULL, the standard default) or
omitted (ABSENT ON NULL).
+ *
+ * A value is spliced in raw (unquoted) when it carries `FORMAT JSON` (tracked
per member by
+ * `rawJson`): the explicit clause (e.g. `JSON_OBJECT('a' VALUE '{"b":1}'
FORMAT JSON)`), or the
+ * implicit form carried by a nested JSON constructor / `JSON_QUERY` under
KEEP QUOTES (see
+ * `rawJsonValue`). Explicit `FORMAT JSON` on a non-constructor is arbitrary
user text, so
+ * `needsValidation` marks it for JSON validation at eval; a nested
constructor is trusted.
+ *
+ * Examples:
+ * JSON_OBJECT('id' VALUE 7, 'name' VALUE 'Ada') ->
'{"id":7,"name":"Ada"}'
+ * JSON_OBJECT('id': 7, 'v': NULL) -> '{"id":7,"v":null}'
+ * JSON_OBJECT('id': 7, 'v': NULL ABSENT ON NULL) -> '{"id":7}'
+ * JSON_OBJECT() -> '{}'
+ *
+ * A clause-free call that is not itself lexically nested in an enclosing
constructor's *value*
+ * position routes through function resolution and is rebuilt by
`JsonObjectExpressionBuilder`, so a
+ * same-named routine can shadow the built-in `json_object`: always for the
comma form (a nested
+ * producer value routes too, carrying its raw-splice eligibility via
`JsonImplicitFormatCarrier`),
+ * but for the `VALUE` / `:` forms only when no member carries a raw value
(`rawJson.contains(true)`
+ * -- an explicit `FORMAT JSON` or a nested producer in *value* position --
keeps it direct).
+ * Otherwise it is built directly from the grammar (clause-bearing, or
value-nested); a nested
+ * *key*-position call stays routable (see `AstBuilder.visitJsonObject`).
+ * The user-facing reference lives in
`docs/sql-ref-syntax-qry-select-json-object.md`.
+ */
+/**
+ * Guards a JSON_OBJECT key: returns the key, or raises JSON_OBJECT_NULL_KEY
when it is null. Wraps
+ * each key inside `JsonObjectExpr.replacement` so that, as the carrier struct
is evaluated left to
+ * right, a null key fails before its member's value is evaluated (the
key-before-value error
+ * order). Internal, so it is `NonSQLExpression`; it never appears in
`JsonObjectExpr.sql`.
+ */
+case class JsonObjectKeyGuard(child: Expression) extends UnaryExpression with
NonSQLExpression {
+ override def dataType: DataType = child.dataType
+ // Can throw, so it is not foldable (mirrors AssertNotNull).
+ override def foldable: Boolean = false
+ // Raises on a null key, so the result is never null.
+ override def nullable: Boolean = false
+
+ override def eval(input: InternalRow): Any = {
+ val value = child.eval(input)
+ if (value == null) {
+ throw QueryExecutionErrors.jsonObjectNullKeyError()
+ }
+ value
+ }
+
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode = {
+ val childGen = child.genCode(ctx)
+ val code = childGen.code + code"""
+ if (${childGen.isNull}) {
+ throw QueryExecutionErrors.jsonObjectNullKeyError();
+ }
+ """
+ ev.copy(code = code, isNull = FalseLiteral, value = childGen.value)
+ }
+
+ override protected def withNewChildInternal(newChild: Expression):
JsonObjectKeyGuard =
+ copy(child = newChild)
+}
+
+case class JsonObjectExpr(
+ members: Seq[(Expression, Expression)],
+ rawJson: Seq[Boolean],
+ needsValidation: Seq[Boolean],
+ nullBehavior: JsonConstructorNullBehavior =
JsonConstructorNullBehavior.Null,
+ returning: DataType = StringType,
+ timeZoneId: Option[String] = None)
+ extends Expression
+ with TimeZoneAwareExpression
+ with QueryErrorsBase
+ // Default RETURNING is a plain STRING, so
`DefaultStringProducingExpression` lets
+ // `ApplyDefaultCollation` cast the result to a non-default collation; the
`dataType` override
+ // below stays authoritative only for an explicitly collated RETURNING.
+ with DefaultStringProducingExpression
+ with ImplicitlyFormattedAsJson
+ with RoutedSqlJsonExpression {
+
+ // `rawJson(i)` splices member `i`'s value verbatim (already-JSON text)
rather than quoting it;
+ // `needsValidation(i)` marks that raw text as arbitrary user input to
JSON-validate at eval (an
+ // explicit `FORMAT JSON` on a non-constructor). Both are frozen at parse
time (see
+ // `AstBuilder.visitJsonObject`). A validated value is always a spliced one.
+ require(members.length == rawJson.length && members.length ==
needsValidation.length,
+ "JsonObjectExpr requires one rawJson and one needsValidation flag per
member")
+ require(needsValidation.lazyZip(rawJson).forall((nv, raw) => !nv || raw),
+ "JsonObjectExpr needsValidation implies rawJson")
+
+ @transient private lazy val memberArray: Array[(Expression, Expression)] =
+ members.toArray
+
+ @transient private lazy val rawJsonArray: Array[Boolean] =
+ rawJson.toArray
+
+ // Throwable only when eval can throw: a key that may be null
(JSON_OBJECT_NULL_KEY), an explicit
+ // `FORMAT JSON` value to validate, or a throwable child. This keeps such
shapes from being pushed
+ // below a filtering join.
+ override lazy val throwable: Boolean =
+ children.exists(_.throwable) || needsValidation.contains(true) ||
+ members.exists { case (k, _) => !k.resolved || k.nullable }
+
+ // The value is never null, but report nullable as `throwable` (like
JsonArray): `NullPropagation`
+ // keys off `nullable`, so a non-nullable constructor would let it fold `IS
[NOT] NULL` to a
+ // constant or `count(...)` to `count(1)`, skipping the eval that must raise
JSON_OBJECT_NULL_KEY.
+ override def nullable: Boolean = throwable
+
+ // Only non-throwing shapes fold, so `ConstantFolding` cannot raise an error
at optimization for
+ // rows a filter would later drop.
+ override def foldable: Boolean = children.forall(_.foldable) && !throwable
+
+ override def dataType: DataType = returning
+
+ // Flattened `[k0, v0, k1, v1, ...]` view, cached since `children` is walked
repeatedly.
+ @transient private lazy val childrenSeq: Seq[Expression] =
+ memberArray.flatMap { case (k, v) => Seq(k, v) }.toImmutableArraySeq
+
+ override def children: Seq[Expression] = childrenSeq
+
+ override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
+ copy(timeZoneId = Option(timeZoneId))
+
+ override def checkInputDataTypes(): TypeCheckResult = {
+ val inputCheck = super.checkInputDataTypes()
+ if (inputCheck.isFailure) {
+ inputCheck
+ } else if (!JsonObjectExpr.isValidReturningType(returning)) {
+ DataTypeMismatch(
+ errorSubClass = "INVALID_JSON_RETURNING_TYPE",
+ messageParameters = Map(
+ "functionName" -> toSQLId(prettyName),
+ "returningType" -> toSQLType(returning)))
+ } else {
+ // Keys must be character strings (they become JSON object member
names). Reject any other
+ // concrete type up front so a non-string key cannot reach the
StringType Jackson writer and
+ // fail with an internal ClassCastException. NullType is allowed
through: a null key value is
+ // reported at runtime as JSON_OBJECT_NULL_KEY rather than a type error.
+ val keyCheck = memberArray.iterator.map(_._1).zipWithIndex.collectFirst {
+ case (k, keyIndex) if !(k.dataType.isInstanceOf[StringType] ||
k.dataType == NullType) =>
+ DataTypeMismatch(
+ errorSubClass = "UNEXPECTED_INPUT_TYPE",
+ messageParameters = Map(
+ "paramIndex" -> ordinalNumber(keyIndex * 2),
+ "requiredType" -> toSQLType(StringType),
+ "inputSql" -> toSQLExpr(k),
+ "inputType" -> toSQLType(k.dataType)))
+ }.getOrElse(TypeCheckResult.TypeCheckSuccess)
+ if (keyCheck.isFailure) {
+ keyCheck
+ } else {
+ // Every value must be serializable to JSON. This mirrors `to_json`'s
analysis-time
+ // `JacksonUtils.verifyType` check, plus a guard for the spatial
atomics that check accepts
+ // but `JacksonGenerator` cannot write (rejecting a runtime failure at
analysis instead).
+ memberArray.iterator.map(_._2).zipWithIndex
+ .foldLeft(TypeCheckResult.TypeCheckSuccess: TypeCheckResult) {
+ case (acc, _) if acc.isFailure => acc
+ case (_, (v, valueIndex)) if rawJsonArray(valueIndex) &&
+ !(v.dataType.isInstanceOf[StringType] || v.dataType ==
NullType) =>
+ // A raw-spliced value must carry JSON text (string); NullType
is allowed (eval drops
+ // or emits it before splicing). Reject any other type up front,
matching JSON_ARRAY's
+ // FORMAT JSON check, rather than failing with a
ClassCastException at eval.
+ DataTypeMismatch(
+ errorSubClass = "INVALID_JSON_FORMAT_JSON_INPUT",
+ messageParameters = Map(
+ "functionName" -> toSQLId(prettyName),
+ // Flattened 1-based arg position, not the member ordinal
(value of member i is
+ // arg 2i+2), matching the key check's
`ordinalNumber(keyIndex*2)`.
+ "position" -> (valueIndex * 2 + 2).toString,
+ "inputType" -> toSQLType(v.dataType)))
+ case (_, (v, _)) =>
+ val elemCheck = JacksonUtils.verifyType(prettyName, v.dataType)
+ if (elemCheck.isFailure) {
+ elemCheck
+ } else if
(JsonObjectExpr.containsUnsupportedJsonType(v.dataType)) {
+ DataTypeMismatch(
+ errorSubClass = "CANNOT_CONVERT_TO_JSON",
+ messageParameters = Map(
+ "name" -> toSQLId(prettyName),
+ "type" -> toSQLType(v.dataType)))
+ } else {
+ TypeCheckResult.TypeCheckSuccess
+ }
+ }
+ }
+ }
+ }
+
+ // The object evaluator holds a mutable JacksonGenerator, so each copy must
be task-local: report
+ // stateful so projections fresh-copy it (codegen embeds the evaluator per
task via its `Invoke`).
+ override def stateful: Boolean = true
+
+ @transient private lazy val resolvedZoneId: String =
+ timeZoneId.getOrElse(SQLConf.get.sessionLocalTimeZone)
+
+ // Writes the whole object through one JacksonGenerator lifecycle (see
[[JsonObjectEvaluator]]).
+ @transient private lazy val objectEvaluator: JsonObjectEvaluator =
JsonObjectEvaluator(
+ memberArray.map(_._2.dataType).toImmutableArraySeq,
+ rawJson,
+ needsValidation,
+ memberArray.map(_._2.foldable).toImmutableArraySeq,
+ nullBehavior == JsonConstructorNullBehavior.Null,
+ resolvedZoneId,
+ prettyName)
+
+ // Render through an `Invoke` of the object evaluator (like `to_json`)
rather than mixing in
+ // `CodegenFallback`, which would drop the enclosing Project from
whole-stage codegen. The members
+ // are carried as a struct `[key0, value0, ...]`; evaluating it left to
right runs each child once
+ // in order, with a `JsonObjectKeyGuard` on each key raising
JSON_OBJECT_NULL_KEY before that
+ // member's value. The node stays in the plan (not `RuntimeReplaceable`) so
its `foldable` and
+ // `throwable` overrides still gate constant-folding and predicate pushdown.
+ @transient private lazy val replacement: Expression = {
+ val carrierFields = memberArray.iterator.zipWithIndex.flatMap { case ((k,
v), i) =>
+ Seq(Literal(s"k$i"), JsonObjectKeyGuard(k), Literal(s"v$i"), v)
+ }.toSeq
+ val carrier = CreateNamedStruct(carrierFields)
+ val render = Invoke(
+ Literal.create(objectEvaluator,
ObjectType(classOf[JsonObjectEvaluator])),
+ "evaluate",
+ StringType,
+ Seq(carrier),
+ Seq(carrier.dataType))
+ if (returning.eq(StringType)) render else Cast(render, returning,
timeZoneId, EvalMode.ANSI)
+ }
+
+ override def eval(input: InternalRow): Any = replacement.eval(input)
+
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode = {
+ val gen = replacement.genCode(ctx)
+ ev.copy(code = gen.code, isNull = gen.isNull, value = gen.value)
+ }
+
+ override def prettyName: String = "json_object"
+
+ override def sql: String = sqlString(forceBuiltinOwnership = true, _.sql)
+
+ // See [[RoutedSqlJsonExpression]] for `forceBuiltinOwnership` and
`renderChild`.
+ private[sql] def sqlString(
+ forceBuiltinOwnership: Boolean,
+ renderChild: Expression => String): String = {
+ val membersSQL = members.zip(rawJson).map { case ((k, v), raw) =>
+ // Render SQL that reparses to the frozen `raw` flag, regardless of
optimizer rewrites of the
+ // child:
+ // - raw + child still a bare implicit producer (bare, or under a
pass-through COLLATE):
+ // render that bare constructor so reparse re-derives implicit raw
splicing.
+ // - raw + child no longer recoverable as a constructor
(constant-folded to a literal, or a
+ // Cast wrapper): render an explicit `FORMAT JSON` so reparse splices
it raw.
+ // - quoted + child would reparse as implicit JSON (an
optimizer-inlined constructor, bare or
+ // under COLLATE): neutralize with `CAST(... AS STRING)` so reparse
keeps it quoted.
+ // - otherwise the child's shape already reproduces the flag.
+ val valueSQL = (raw, JsonObjectExpr.rawJsonValue(v)) match {
+ case (true, Some(c)) => renderChild(c)
+ case (true, None) => s"${renderChild(v)} FORMAT JSON"
+ case (false, Some(_)) => s"CAST(${renderChild(v)} AS STRING)"
+ case (false, None) => renderChild(v)
+ }
+ s"${renderChild(k)} VALUE $valueSQL"
+ }.mkString(", ")
+ // Render the null-handling clause. A non-default `ABSENT ON NULL` always
renders (it is part of
+ // the requested semantics). The default `NULL ON NULL` is synthetic here:
canonical `sql`
+ // (`forceBuiltinOwnership`) appends it to force the direct grammar path
on reparse (a
+ // clause-free call routes through function resolution and could bind a
shadowing routine), so a
+ // resolved built-in round-trips to the same built-in; the never-reparsed
pretty form omits it
+ // so column names do not expose ownership text the user never wrote.
Mirrors `JsonArray`.
+ val nullSQL = nullBehavior match {
+ case JsonConstructorNullBehavior.Absent => " ABSENT ON NULL"
+ case JsonConstructorNullBehavior.Null => if (forceBuiltinOwnership) "
NULL ON NULL" else ""
+ }
+ // Reference identity, not value equality: an explicit `RETURNING STRING
COLLATE ...` is a
+ // distinct StringType instance that `==` the default companion, so `==`
would drop it. `eq`
Review Comment:
Done — narrowed the comment to `COLLATE UTF8_BINARY` to match the sibling
`sql` renderers.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
