rednaxelafx commented on a change in pull request #28463:
URL: https://github.com/apache/spark/pull/28463#discussion_r421913776
##########
File path: core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
##########
@@ -414,6 +434,284 @@ private[spark] object ClosureCleaner extends Logging {
}
}
+private[spark] object IndylambdaScalaClosures extends Logging {
+ // internal name of java.lang.invoke.LambdaMetafactory
+ val LambdaMetafactoryClassName = "java/lang/invoke/LambdaMetafactory"
+ // the method that Scala indylambda use for bootstrap method
+ val LambdaMetafactoryMethodName = "altMetafactory"
+ val LambdaMetafactoryMethodDesc = "(Ljava/lang/invoke/MethodHandles$Lookup;"
+
+ "Ljava/lang/String;Ljava/lang/invoke/MethodType;[Ljava/lang/Object;)" +
+ "Ljava/lang/invoke/CallSite;"
+
+ /**
+ * Check if the given reference is a indylambda style Scala closure.
+ * If so, return a non-empty serialization proxy (SerializedLambda) of the
closure;
+ * otherwise return None.
+ *
+ * @param maybeClosure the closure to check.
+ */
+ def getSerializationProxy(maybeClosure: AnyRef): Option[SerializedLambda] = {
+ val maybeClosureClass = maybeClosure.getClass
+
+ // shortcut the fast check:
+ // 1. indylambda closure classes are generated by Java's
LambdaMetafactory, and they're always
+ // synthetic.
+ // 2. We only care about Serializable closures, so let's check that as well
+ if (!maybeClosureClass.isSynthetic ||
!maybeClosure.isInstanceOf[Serializable]) return None
+
+ val implementedInterfaces =
ClassUtils.getAllInterfaces(maybeClosureClass).asScala
+ val isClosureCandidate =
implementedInterfaces.exists(_.getName.startsWith("scala.Function"))
+
+ if (isClosureCandidate) {
+ try {
+ val lambdaProxy = inspect(maybeClosure)
+ if (isIndylambdaScalaClosure(lambdaProxy)) Option(lambdaProxy)
+ else None
+ } catch {
+ case e: Exception =>
+ // no need to check if debug is enabled here the Spark logging api
covers this.
+ logDebug("The given reference is not an indylambda Scala closure.",
e)
+ None
+ }
+ } else {
+ None
+ }
+ }
+
+ def isIndylambdaScalaClosure(lambdaProxy: SerializedLambda): Boolean = {
+ lambdaProxy.getImplMethodKind == MethodHandleInfo.REF_invokeStatic &&
+ lambdaProxy.getImplMethodName.contains("$anonfun$")
+ }
+
+ def inspect(closure: AnyRef): SerializedLambda = {
+ val writeReplace = closure.getClass.getDeclaredMethod("writeReplace")
+ writeReplace.setAccessible(true)
+ writeReplace.invoke(closure).asInstanceOf[SerializedLambda]
+ }
+
+ /**
+ * Check if the handle represents the LambdaMetafactory that indylambda
Scala closures
+ * use for creating the lambda class and getting a closure instance.
+ */
+ def isLambdaMetafactory(bsmHandle: Handle): Boolean = {
+ bsmHandle.getOwner == LambdaMetafactoryClassName &&
+ bsmHandle.getName == LambdaMetafactoryMethodName &&
+ bsmHandle.getDesc == LambdaMetafactoryMethodDesc
+ }
+
+ /**
+ * Check if the handle represents a target method that is:
+ * - a STATIC method that implements a Scala lambda body in the indylambda
style
+ * - captures the enclosing `this`, i.e. the first argument is a reference
to the same type as
+ * the owning class.
+ * Returns true if both criteria above are met.
+ */
+ def isLambdaBodyCapturingOuter(handle: Handle, ownerInternalName: String):
Boolean = {
+ handle.getTag == H_INVOKESTATIC &&
+ handle.getName.contains("$anonfun$") &&
+ handle.getOwner == ownerInternalName &&
+ handle.getDesc.startsWith(s"(L$ownerInternalName;")
+ }
+
+ /**
+ * Check if the callee of a call site is a inner class constructor.
+ * - A constructor has to be invoked via INVOKESPECIAL
+ * - A constructor's internal name is "<init>" and the return type is "V"
(void)
+ * - An inner class' first argument in the signature has to be a reference
to the
+ * enclosing "this", aka `$outer` in Scala.
+ */
+ def isInnerClassCtorCapturingOuter(
+ op: Int, owner: String, name: String, desc: String, callerInternalName:
String): Boolean = {
+ op == INVOKESPECIAL && name == "<init>" &&
desc.startsWith(s"(L$callerInternalName;")
+ }
+
+ // scalastyle:off line.size.limit
+ /**
+ * Scans an indylambda Scala closure, along with its lexically nested
closures, and populate
+ * the accessed fields info on which fields on the outer object are accessed.
+ *
+ * Example: run the following code snippet in a Spark Shell w/ Scala 2.12+:
Review comment:
Hi @retronym , I've added an example with trace-level logs here. Does
this look good to you, or would you consider this too verbose?
The actual log produced is the following:
```
20/05/07 19:49:27 DEBUG ClosureCleaner: Cleaning indylambda closure:
$anonfun$closure$1$adapted
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: scanning
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.$anonfun$closure$1$adapted(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw;Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
scala.runtime.BoxesRunTime.unboxToInt(Ljava/lang/Object;)I
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: found intra class call
to
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.$anonfun$closure$1(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw;I)Lscala/collection/immutable/IndexedSeq;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: scanning
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.$anonfun$closure$1(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw;I)Lscala/collection/immutable/IndexedSeq;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: found inner class
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
scala.Predef$.intWrapper(I)I
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
scala.runtime.RichInt$.to$extension0(II)Lscala/collection/immutable/Range$Inclusive;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: found call to outer
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.innerClosure()Lscala/Function1;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
scala.collection.immutable.IndexedSeq$.canBuildFrom()Lscala/collection/generic/CanBuildFrom;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
scala.collection.immutable.Range$Inclusive.flatMap(Lscala/Function1;Lscala/collection/generic/CanBuildFrom;)Ljava/lang/Object;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: scanning
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.innerClosure()Lscala/Function1;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: scanning
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$deserializeLambda$(Ljava/lang/invoke/SerializedLambda;)Ljava/lang/Object;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: invokedynamic:
lambdaDeserialize(Ljava/lang/invoke/SerializedLambda;)Ljava/lang/Object;,
bsmHandle=scala/runtime/LambdaDeserialize.bootstrap(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/invoke/MethodType;[Ljava/lang/invoke/MethodHandle;)Ljava/lang/invoke/CallSite;
(6),
bsmArgs=WrappedArray($line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$2$adapted(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String;
(6),
$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$1$adapted(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq;
(6))
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: scanning
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$1$adapted(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
scala.runtime.BoxesRunTime.unboxToInt(Ljava/lang/Object;)I
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: found intra class call
to
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$1(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;I)Lscala/collection/immutable/IndexedSeq;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: scanning
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$1(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;I)Lscala/collection/immutable/IndexedSeq;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
scala.Predef$.intWrapper(I)I
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
scala.runtime.RichInt$.to$extension0(II)Lscala/collection/immutable/Range$Inclusive;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: invokedynamic:
apply(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;)Lscala/Function1;,
bsmHandle=java/lang/invoke/LambdaMetafactory.altMetafactory(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/invoke/MethodType;[Ljava/lang/Object;)Ljava/lang/invoke/CallSite;
(6), bsmArgs=WrappedArray((Ljava/lang/Object;)Ljava/lang/Object;,
$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$2$adapted(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String;
(6), (Ljava/lang/Object;)Ljava/lang/String;, 7, 1, Lscala/Serializable;, 1,
(Ljava/lang/Object;)Ljava/lang/String;)
20/05/07 19:49:27 DEBUG IndylambdaScalaClosures: found inner closure
$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$2$adapted(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String;
(6)
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
scala.collection.immutable.IndexedSeq$.canBuildFrom()Lscala/collection/generic/CanBuildFrom;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
scala.collection.immutable.Range$Inclusive.map(Lscala/Function1;Lscala/collection/generic/CanBuildFrom;)Ljava/lang/Object;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: scanning
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$2$adapted(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
scala.runtime.BoxesRunTime.unboxToInt(Ljava/lang/Object;)I
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: found intra class call
to
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$2(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;I)Ljava/lang/String;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: scanning
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$2(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;I)Ljava/lang/String;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
java.lang.StringBuilder.<init>(I)V
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
java.lang.StringBuilder.append(I)Ljava/lang/StringBuilder;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: found call to outer
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.topLevelValue()Ljava/lang/String;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
java.lang.StringBuilder.append(Ljava/lang/String;)Ljava/lang/StringBuilder;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
java.lang.StringBuilder.toString()Ljava/lang/String;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: scanning
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.topLevelValue()Ljava/lang/String;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: found field access
topLevelValue on $line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: scanning
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$2$adapted(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
scala.runtime.BoxesRunTime.unboxToInt(Ljava/lang/Object;)I
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: found intra class call
to
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$2(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;I)Ljava/lang/String;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: scanning
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.<init>(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw;)V
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
java.lang.Object.<init>()V
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: invokedynamic:
apply(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;)Lscala/Function1;,
bsmHandle=java/lang/invoke/LambdaMetafactory.altMetafactory(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/invoke/MethodType;[Ljava/lang/Object;)Ljava/lang/invoke/CallSite;
(6), bsmArgs=WrappedArray((Ljava/lang/Object;)Ljava/lang/Object;,
$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$1$adapted(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq;
(6), (Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq;, 7, 1,
Lscala/Serializable;, 1,
(Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq;)
20/05/07 19:49:27 DEBUG IndylambdaScalaClosures: found inner closure
$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$1$adapted(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq;
(6)
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: scanning
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$1(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;I)Lscala/collection/immutable/IndexedSeq;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
scala.Predef$.intWrapper(I)I
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
scala.runtime.RichInt$.to$extension0(II)Lscala/collection/immutable/Range$Inclusive;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: invokedynamic:
apply(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;)Lscala/Function1;,
bsmHandle=java/lang/invoke/LambdaMetafactory.altMetafactory(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/invoke/MethodType;[Ljava/lang/Object;)Ljava/lang/invoke/CallSite;
(6), bsmArgs=WrappedArray((Ljava/lang/Object;)Ljava/lang/Object;,
$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$2$adapted(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String;
(6), (Ljava/lang/Object;)Ljava/lang/String;, 7, 1, Lscala/Serializable;, 1,
(Ljava/lang/Object;)Ljava/lang/String;)
20/05/07 19:49:27 DEBUG IndylambdaScalaClosures: found inner closure
$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$2$adapted(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String;
(6)
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
scala.collection.immutable.IndexedSeq$.canBuildFrom()Lscala/collection/generic/CanBuildFrom;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
scala.collection.immutable.Range$Inclusive.map(Lscala/Function1;Lscala/collection/generic/CanBuildFrom;)Ljava/lang/Object;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: scanning
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.$anonfun$innerClosure$2(L$line16/$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1;I)Ljava/lang/String;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
java.lang.StringBuilder.<init>(I)V
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
java.lang.StringBuilder.append(I)Ljava/lang/StringBuilder;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: found call to outer
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.topLevelValue()Ljava/lang/String;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
java.lang.StringBuilder.append(Ljava/lang/String;)Ljava/lang/StringBuilder;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: ignoring call to
java.lang.StringBuilder.toString()Ljava/lang/String;
20/05/07 19:49:27 TRACE IndylambdaScalaClosures: scanning
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$InnerFoo$1.innerClosure()Lscala/Function1;
20/05/07 19:49:27 DEBUG ClosureCleaner: + fields accessed by starting
closure: 2 classes
20/05/07 19:49:27 DEBUG ClosureCleaner: (class java.lang.Object,Set())
20/05/07 19:49:27 DEBUG ClosureCleaner: (class
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw,Set(topLevelValue))
20/05/07 19:49:27 DEBUG ClosureCleaner: + cloning instance of REPL class
$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw
20/05/07 19:49:27 DEBUG ClosureCleaner: +++ indylambda closure
($anonfun$closure$1$adapted) is now cleaned +++
20/05/07 19:49:27 DEBUG ClosureCleaner: Cleaning indylambda closure:
$anonfun$collect$2
20/05/07 19:49:27 DEBUG ClosureCleaner: +++ indylambda closure
($anonfun$collect$2) is now cleaned +++
20/05/07 19:49:27 DEBUG ClosureCleaner: Cleaning indylambda closure:
$anonfun$runJob$5
20/05/07 19:49:27 DEBUG ClosureCleaner: +++ indylambda closure
($anonfun$runJob$5) is now cleaned +++
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
