cloud-fan commented on code in PR #57497:
URL: https://github.com/apache/spark/pull/57497#discussion_r3749681600
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtils.scala:
##########
@@ -280,6 +280,70 @@ object VariantExpressionEvalUtils {
new VariantVal(v.getValue, v.getMetadata)
}
+ /**
+ * Build a variant object directly from a keys array and a values array,
without materializing an
+ * intermediate map. Keys must be non-null strings and the two arrays must
have equal length. A
+ * null key raises `NULL_MAP_KEY`, a duplicate key raises
`VARIANT_DUPLICATE_KEY` (matching
+ * to_variant_object), and null values are kept as variant null.
+ */
+ def variantFromArrays(keys: ArrayData, values: ArrayData, valueType:
DataType): VariantVal = {
+ if (keys.numElements() != values.numElements()) {
+ // Reuse the same error map_from_arrays raises for a keys/values length
mismatch.
+ throw
QueryExecutionErrors.mapDataKeyArrayLengthDiffersFromValueArrayLengthError()
+ }
+ val builder = new VariantBuilder(false)
+ val start = builder.getWritePos
+ val numElements = keys.numElements()
+ val fields = new
java.util.ArrayList[VariantBuilder.FieldEntry](numElements)
+ var i = 0
+ while (i < numElements) {
+ if (keys.isNullAt(i)) {
+ throw QueryExecutionErrors.nullAsMapKeyNotAllowedError()
+ }
+ val key = keys.getUTF8String(i).toString
+ val id = builder.addKey(key)
+ fields.add(new VariantBuilder.FieldEntry(key, id, builder.getWritePos -
start))
+ val value = if (values.isNullAt(i)) null else values.get(i, valueType)
+ buildVariant(builder, value, valueType)
+ i += 1
+ }
+ builder.finishWritingObject(start, fields)
+ val v = builder.result()
+ new VariantVal(v.getValue, v.getMetadata)
+ }
+
+ /**
+ * Build a variant object directly from an array of key/value struct
entries, without an
+ * intermediate map. Keys must be non-null strings. A null key raises
`NULL_MAP_KEY`, a
+ * duplicate key raises `VARIANT_DUPLICATE_KEY`, null values are kept as
variant null, and a
+ * null entry makes the whole result null.
+ */
+ def variantFromEntries(entries: ArrayData, valueType: DataType): VariantVal
= {
+ val builder = new VariantBuilder(false)
+ val start = builder.getWritePos
+ val numElements = entries.numElements()
+ val fields = new
java.util.ArrayList[VariantBuilder.FieldEntry](numElements)
+ var i = 0
+ while (i < numElements) {
+ if (entries.isNullAt(i)) {
Review Comment:
Please preserve the documented null-entry dominance before converting any
values. In this single-pass loop, an earlier nested value can fail during
`buildVariant` (for example, a struct with duplicate field names) before a
later null entry is reached, so the call throws instead of returning null as
`map_from_entries` does; retaining the pre-scan and adding a mixed
failing-value/null test would keep the contract intact.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
