uros-b commented on code in PR #56864:
URL: https://github.com/apache/spark/pull/56864#discussion_r3735153416
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala:
##########
@@ -1490,6 +1490,60 @@ object VariantArrayAppendExpressionBuilder extends
VariantArrayAppendExpressionB
// scalastyle:on line.size.limit
object TryVariantArrayAppendExpressionBuilder
extends VariantArrayAppendExpressionBuilderBase(false)
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+ usage = "_FUNC_(v[, includeArrays]) - Recursively removes null fields from
variant objects " +
+ "and null elements from arrays, unless `includeArrays` is false, in which
case null " +
+ "elements in arrays are kept. Returns NULL if any argument is NULL.",
+ arguments = """
+ Arguments:
+ * v - A variant value to mutate.
+ * includeArrays - An optional boolean (default true).
+ """,
+ examples = """
+ Examples:
+ > SELECT _FUNC_(parse_json('{"a": 1, "b": null, "c": 3}'));
+ {"a":1,"c":3}
+ > SELECT _FUNC_(parse_json('[1, null, 3]'));
+ [1,3]
+ > SELECT _FUNC_(parse_json('{"a": {"b": null, "c": [1, null]}}'));
+ {"a":{"c":[1]}}
+ > SELECT _FUNC_(parse_json('{"a": [1, null], "b": null}'), false);
+ {"a":[1,null]}
+ > SELECT _FUNC_(NULL);
+ NULL
+ """,
+ since = "4.3.0",
+ group = "variant_funcs"
+)
+// scalastyle:on line.size.limit
+case class VariantStripNulls(child: Expression, includeArrays: Expression)
+ extends RuntimeReplaceable
+ with ExpectsInputTypes
+ with BinaryLike[Expression] {
+
+ def this(child: Expression) = this(child, Literal(true))
+
+ override def left: Expression = child
+ override def right: Expression = includeArrays
+
+ override def dataType: DataType = VariantType
+ override def inputTypes: Seq[AbstractDataType] = Seq(VariantType,
BooleanType)
+
+ override lazy val replacement: Expression = StaticInvoke(
+ VariantExpressionEvalUtils.getClass,
+ VariantType,
Review Comment:
We probably don't need the dataType override then
(https://github.com/apache/spark/pull/56864/changes#r3735150847).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
