This is an automated email from the ASF dual-hosted git repository. yangjie01 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 08c49637795f [SPARK-47770][INFRA] Fix `GenerateMIMAIgnore.isPackagePrivateModule` to return `false` instead of failing 08c49637795f is described below commit 08c49637795fd56ef550a509648f0890ff22a948 Author: Dongjoon Hyun <dh...@apple.com> AuthorDate: Tue Apr 9 11:14:49 2024 +0800 [SPARK-47770][INFRA] Fix `GenerateMIMAIgnore.isPackagePrivateModule` to return `false` instead of failing ### What changes were proposed in this pull request? This PR aims to fix `GenerateMIMAIgnore.isPackagePrivateModule` to work correctly. For example, `Metadata` is a case class inside package private `DefaultParamsReader` class. Currently, MIMA fails at this class analysis. https://github.com/apache/spark/blob/f8e652e88320528a70e605a6a3cf986725e153a5/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala#L474-L485 The root cause is `isPackagePrivateModule` fails due to `scala.ScalaReflectionException`. We can simply make `isPackagePrivateModule` return `false` instead of failing. ``` Error instrumenting class:org.apache.spark.ml.util.DefaultParamsReader$Metadata Exception in thread "main" scala.ScalaReflectionException: type Serializable is not a class at scala.reflect.api.Symbols$SymbolApi.asClass(Symbols.scala:284) at scala.reflect.api.Symbols$SymbolApi.asClass$(Symbols.scala:284) at scala.reflect.internal.Symbols$SymbolContextApiImpl.asClass(Symbols.scala:99) at scala.reflect.runtime.JavaMirrors$JavaMirror.classToScala1(JavaMirrors.scala:1085) at scala.reflect.runtime.JavaMirrors$JavaMirror.$anonfun$classToScala$1(JavaMirrors.scala:1040) at scala.reflect.runtime.JavaMirrors$JavaMirror.$anonfun$toScala$1(JavaMirrors.scala:150) at scala.reflect.runtime.TwoWayCaches$TwoWayCache.toScala(TwoWayCaches.scala:50) at scala.reflect.runtime.JavaMirrors$JavaMirror.toScala(JavaMirrors.scala:148) at scala.reflect.runtime.JavaMirrors$JavaMirror.classToScala(JavaMirrors.scala:1040) at scala.reflect.runtime.JavaMirrors$JavaMirror.typeToScala(JavaMirrors.scala:1148) at scala.reflect.runtime.JavaMirrors$JavaMirror$FromJavaClassCompleter.$anonfun$completeRest$2(JavaMirrors.scala:816) at scala.reflect.runtime.JavaMirrors$JavaMirror$FromJavaClassCompleter.$anonfun$completeRest$1(JavaMirrors.scala:816) at scala.reflect.runtime.JavaMirrors$JavaMirror$FromJavaClassCompleter.completeRest(JavaMirrors.scala:810) at scala.reflect.runtime.JavaMirrors$JavaMirror$FromJavaClassCompleter.complete(JavaMirrors.scala:806) at scala.reflect.internal.Symbols$Symbol.completeInfo(Symbols.scala:1575) at scala.reflect.internal.Symbols$Symbol.info(Symbols.scala:1538) at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol$$anon$13.scala$reflect$runtime$SynchronizedSymbols$SynchronizedSymbol$$super$info(SynchronizedSymbols.scala:221) at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol.info(SynchronizedSymbols.scala:158) at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol.info$(SynchronizedSymbols.scala:158) at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol$$anon$13.info(SynchronizedSymbols.scala:221) at scala.reflect.internal.Symbols$Symbol.initialize(Symbols.scala:1733) at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol.privateWithin(SynchronizedSymbols.scala:109) at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol.privateWithin$(SynchronizedSymbols.scala:107) at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol$$anon$13.privateWithin(SynchronizedSymbols.scala:221) at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol$$anon$13.privateWithin(SynchronizedSymbols.scala:221) at org.apache.spark.tools.GenerateMIMAIgnore$.isPackagePrivateModule(GenerateMIMAIgnore.scala:48) at org.apache.spark.tools.GenerateMIMAIgnore$.$anonfun$privateWithin$1(GenerateMIMAIgnore.scala:67) at scala.collection.immutable.List.foreach(List.scala:334) at org.apache.spark.tools.GenerateMIMAIgnore$.privateWithin(GenerateMIMAIgnore.scala:61) at org.apache.spark.tools.GenerateMIMAIgnore$.main(GenerateMIMAIgnore.scala:125) at org.apache.spark.tools.GenerateMIMAIgnore.main(GenerateMIMAIgnore.scala) ``` ### Why are the changes needed? **BEFORE** ``` $ dev/mima | grep org.apache.spark.ml.util.DefaultParamsReader Using SPARK_LOCAL_IP=localhost Using SPARK_LOCAL_IP=localhost Error instrumenting class:org.apache.spark.ml.util.DefaultParamsReader$Metadata$ Error instrumenting class:org.apache.spark.ml.util.DefaultParamsReader$Metadata Using SPARK_LOCAL_IP=localhost # I checked the following before deleing `.generated-mima-class-excludes ` $ cat .generated-mima-class-excludes | grep org.apache.spark.ml.util.DefaultParamsReader org.apache.spark.ml.util.DefaultParamsReader$ org.apache.spark.ml.util.DefaultParamsReader# org.apache.spark.ml.util.DefaultParamsReader ``` **AFTER** ``` $ dev/mima | grep org.apache.spark.ml.util.DefaultParamsReader Using SPARK_LOCAL_IP=localhost Using SPARK_LOCAL_IP=localhost [WARN] Unable to detect inner functions for class:org.apache.spark.ml.util.DefaultParamsReader.Metadata [WARN] Unable to detect inner functions for class:org.apache.spark.ml.util.DefaultParamsReader.Metadata Using SPARK_LOCAL_IP=localhost # I checked the following before deleting `.generated-mima-class-excludes `. $ cat .generated-mima-class-excludes | grep org.apache.spark.ml.util.DefaultParamsReader org.apache.spark.ml.util.DefaultParamsReader$Metadata$ org.apache.spark.ml.util.DefaultParamsReader$ org.apache.spark.ml.util.DefaultParamsReader#Metadata# org.apache.spark.ml.util.DefaultParamsReader# org.apache.spark.ml.util.DefaultParamsReader$Metadata org.apache.spark.ml.util.DefaultParamsReader#Metadata org.apache.spark.ml.util.DefaultParamsReader ``` ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Manual tests. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #45938 from dongjoon-hyun/SPARK-47770. Authored-by: Dongjoon Hyun <dh...@apple.com> Signed-off-by: yangjie01 <yangji...@baidu.com> --- .../main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala index acab365418af..9938a575f08f 100644 --- a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala +++ b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala @@ -44,8 +44,15 @@ object GenerateMIMAIgnore { private def isPackagePrivate(sym: unv.Symbol) = !sym.privateWithin.fullName.startsWith("<none>") - private def isPackagePrivateModule(moduleSymbol: unv.ModuleSymbol) = + private def isPackagePrivateModule(moduleSymbol: unv.ModuleSymbol) = try { !moduleSymbol.privateWithin.fullName.startsWith("<none>") + } catch { + case e: Throwable => + // scalastyle:off println + println("[WARN] Unable to check module:" + moduleSymbol) + // scalastyle:on println + false + } /** * For every class checks via scala reflection if the class itself or contained members --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org