hvanhovell commented on code in PR #41284:
URL: https://github.com/apache/spark/pull/41284#discussion_r1244565197
##########
connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala:
##########
@@ -122,65 +125,62 @@ object CheckConnectJvmClientCompatibility {
private def checkMiMaCompatibilityWithSqlModule(
clientJar: File,
sqlJar: File): List[Problem] = {
- val includedRules = Seq(
- IncludeByName("org.apache.spark.sql.catalog.Catalog.*"),
- IncludeByName("org.apache.spark.sql.catalog.CatalogMetadata.*"),
- IncludeByName("org.apache.spark.sql.catalog.Column.*"),
- IncludeByName("org.apache.spark.sql.catalog.Database.*"),
- IncludeByName("org.apache.spark.sql.catalog.Function.*"),
- IncludeByName("org.apache.spark.sql.catalog.Table.*"),
- IncludeByName("org.apache.spark.sql.Column.*"),
- IncludeByName("org.apache.spark.sql.ColumnName.*"),
- IncludeByName("org.apache.spark.sql.DataFrame.*"),
- IncludeByName("org.apache.spark.sql.DataFrameReader.*"),
- IncludeByName("org.apache.spark.sql.DataFrameNaFunctions.*"),
- IncludeByName("org.apache.spark.sql.DataFrameStatFunctions.*"),
- IncludeByName("org.apache.spark.sql.DataFrameWriter.*"),
- IncludeByName("org.apache.spark.sql.DataFrameWriterV2.*"),
- IncludeByName("org.apache.spark.sql.Dataset.*"),
- IncludeByName("org.apache.spark.sql.functions.*"),
- IncludeByName("org.apache.spark.sql.KeyValueGroupedDataset.*"),
- IncludeByName("org.apache.spark.sql.RelationalGroupedDataset.*"),
- IncludeByName("org.apache.spark.sql.SparkSession.*"),
- IncludeByName("org.apache.spark.sql.RuntimeConfig.*"),
- IncludeByName("org.apache.spark.sql.TypedColumn.*"),
- IncludeByName("org.apache.spark.sql.SQLImplicits.*"),
- IncludeByName("org.apache.spark.sql.DatasetHolder.*"),
- IncludeByName("org.apache.spark.sql.streaming.DataStreamReader.*"),
- IncludeByName("org.apache.spark.sql.streaming.DataStreamWriter.*"),
- IncludeByName("org.apache.spark.sql.streaming.StreamingQuery.*"),
-
IncludeByName("org.apache.spark.sql.streaming.StreamingQueryManager.active"),
-
IncludeByName("org.apache.spark.sql.streaming.StreamingQueryManager.get"),
-
IncludeByName("org.apache.spark.sql.streaming.StreamingQueryManager.awaitAnyTermination"),
-
IncludeByName("org.apache.spark.sql.streaming.StreamingQueryManager.resetTerminated"),
- IncludeByName("org.apache.spark.sql.streaming.StreamingQueryStatus.*"),
- IncludeByName("org.apache.spark.sql.streaming.StreamingQueryProgress.*"))
+ val includedRules = Seq(IncludeByName("org.apache.spark.sql.*"))
val excludeRules = Seq(
// Filter unsupported rules:
// Note when muting errors for a method, checks on all overloading
methods are also muted.
- // Skip all shaded dependencies and proto files in the client.
- ProblemFilters.exclude[Problem]("org.sparkproject.*"),
- ProblemFilters.exclude[Problem]("org.apache.spark.connect.proto.*"),
+ // Skip unsupported packages
+ ProblemFilters.exclude[Problem]("org.apache.spark.sql.api.*"), // Java,
Python, R
+ ProblemFilters.exclude[Problem]("org.apache.spark.sql.catalyst.*"),
+ ProblemFilters.exclude[Problem]("org.apache.spark.sql.columnar.*"),
+ ProblemFilters.exclude[Problem]("org.apache.spark.sql.connector.*"),
+ ProblemFilters.exclude[Problem]("org.apache.spark.sql.execution.*"),
+ ProblemFilters.exclude[Problem]("org.apache.spark.sql.expressions.*"),
+ ProblemFilters.exclude[Problem]("org.apache.spark.sql.internal.*"),
+ ProblemFilters.exclude[Problem]("org.apache.spark.sql.jdbc.*"),
+ ProblemFilters.exclude[Problem]("org.apache.spark.sql.sources.*"),
+ ProblemFilters.exclude[Problem]("org.apache.spark.sql.streaming.ui.*"),
+ ProblemFilters.exclude[Problem]("org.apache.spark.sql.test.*"),
+ ProblemFilters.exclude[Problem]("org.apache.spark.sql.util.*"),
+
+ // Skip private[sql] constructors
+ ProblemFilters.exclude[Problem]("org.apache.spark.sql.*.this"),
+
+ // Skip unsupported classes
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.ExperimentalMethods"),
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SQLContext"),
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SQLContext$*"),
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SparkSessionExtensions"),
+ ProblemFilters.exclude[MissingClassProblem](
+ "org.apache.spark.sql.SparkSessionExtensionsProvider"),
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.UDTFRegistration"),
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.UDFRegistration"),
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.UDFRegistration$"),
// DataFrame Reader & Writer
-
ProblemFilters.exclude[Problem]("org.apache.spark.sql.DataFrameReader.json"),
// deprecated
+
ProblemFilters.exclude[Problem]("org.apache.spark.sql.DataFrameReader.json"),
// rdd
// DataFrameNaFunctions
-
ProblemFilters.exclude[Problem]("org.apache.spark.sql.DataFrameNaFunctions.this"),
ProblemFilters.exclude[Problem]("org.apache.spark.sql.DataFrameNaFunctions.fillValue"),
// DataFrameStatFunctions
ProblemFilters.exclude[Problem]("org.apache.spark.sql.DataFrameStatFunctions.bloomFilter"),
-
ProblemFilters.exclude[Problem]("org.apache.spark.sql.DataFrameStatFunctions.this"),
// Dataset
+ ProblemFilters.exclude[MissingClassProblem](
+ "org.apache.spark.sql.Dataset$" // private[sql]
+ ),
ProblemFilters.exclude[Problem]("org.apache.spark.sql.Dataset.ofRows"),
ProblemFilters.exclude[Problem]("org.apache.spark.sql.Dataset.DATASET_ID_TAG"),
ProblemFilters.exclude[Problem]("org.apache.spark.sql.Dataset.COL_POS_KEY"),
ProblemFilters.exclude[Problem]("org.apache.spark.sql.Dataset.DATASET_ID_KEY"),
ProblemFilters.exclude[Problem]("org.apache.spark.sql.Dataset.curId"),
ProblemFilters.exclude[Problem]("org.apache.spark.sql.Dataset.observe"),
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.Observation"),
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.Observation$"),
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.ObservationListener"),
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.ObservationListener$"),
ProblemFilters.exclude[Problem]("org.apache.spark.sql.Dataset.queryExecution"),
ProblemFilters.exclude[Problem]("org.apache.spark.sql.Dataset.encoder"),
Review Comment:
Is this a type mismatch?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
