Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/10239#discussion_r47184645
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
---
@@ -326,19 +326,28 @@ case class WrapOption(child: Expression)
* A place holder for the loop variable used in [[MapObjects]]. This
should never be constructed
* manually, but will instead be passed into the provided lambda function.
*/
-case class LambdaVariable(value: String, isNull: String, dataType:
DataType) extends Expression {
+case class LambdaVariable(value: String, isNull: String, dataType:
DataType) extends LeafExpression
+ with Unevaluable {
- override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String =
- throw new UnsupportedOperationException("Only calling gen() is
supported.")
+ override def nullable: Boolean = true
- override def children: Seq[Expression] = Nil
- override def gen(ctx: CodeGenContext): GeneratedExpressionCode =
+ override def gen(ctx: CodeGenContext): GeneratedExpressionCode = {
GeneratedExpressionCode(code = "", value = value, isNull = isNull)
+ }
+}
- override def nullable: Boolean = false
- override def eval(input: InternalRow): Any =
- throw new UnsupportedOperationException("Only code-generated
evaluation is supported.")
+object MapObjects {
+ private val curId = new java.util.concurrent.atomic.AtomicInteger()
+ def apply(
+ function: Expression => Expression,
+ inputData: Expression,
+ elementType: DataType): MapObjects = {
+ val loopValue = "MapObjects_loopValue" + curId.getAndIncrement()
+ val loopIsNull = "MapObjects_loopIsNull" + curId.getAndIncrement()
+ val loopVar = LambdaVariable(loopValue, loopIsNull, elementType)
--- End diff --
Here is a little hacky, but I wanna argue that it's way less hacky than the
previous implementation and I make the variable name verbose to make it less
possible to get conflicted.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
