This is an automated email from the ASF dual-hosted git repository.
hvanhovell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 885e9c111d2 [SPARK-43265] Move Error framework to a common utils module
885e9c111d2 is described below
commit 885e9c111d2ad7a30fc559d26abbd5dbf1d5ef57
Author: Rui Wang <[email protected]>
AuthorDate: Wed Apr 26 21:23:27 2023 -0400
[SPARK-43265] Move Error framework to a common utils module
### What changes were proposed in this pull request?
Move Error framework to a common utils module so that we can share it
between Spark and Spark Connect without introducing heavy dependencies on Spark
Connect module.
### Why are the changes needed?
Reduce Dependencies on Spark Connect.
### Does this PR introduce _any_ user-facing change?
Error framework is internally API so this should be fine.
### How was this patch tested?
Existing UT.
Closes #40931 from amaliujia/SPARK-43265.
Authored-by: Rui Wang <[email protected]>
Signed-off-by: Herman van Hovell <[email protected]>
---
common/utils/pom.xml | 61 ++++++++++++++++++++++
.../main/java/org/apache/spark/QueryContext.java | 0
.../main/java/org/apache/spark/SparkThrowable.java | 0
.../org/apache/spark/ErrorClassesJSONReader.scala | 0
.../scala/org/apache/spark/SparkException.scala | 17 ------
.../org/apache/spark/SparkThrowableHelper.scala | 5 +-
.../scala/org/apache/spark/util/JsonUtils.scala | 41 +++++++++++++++
core/pom.xml | 5 ++
.../spark/SparkFileAlreadyExistsException.scala | 37 +++++++++++++
pom.xml | 1 +
project/MimaExcludes.scala | 6 +++
project/SparkBuild.scala | 8 +--
12 files changed, 158 insertions(+), 23 deletions(-)
diff --git a/common/utils/pom.xml b/common/utils/pom.xml
new file mode 100644
index 00000000000..8d24d888e6d
--- /dev/null
+++ b/common/utils/pom.xml
@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-parent_2.12</artifactId>
+ <version>3.5.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>spark-common-utils_2.12</artifactId>
+ <packaging>jar</packaging>
+ <name>Spark Project Common Utils</name>
+ <url>https://spark.apache.org/</url>
+ <properties>
+ <sbt.project.name>common-utils</sbt.project.name>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ <artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-text</artifactId>
+ </dependency>
+ </dependencies>
+ <build>
+
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
+
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+ <plugins>
+ </plugins>
+ </build>
+</project>
diff --git a/core/src/main/java/org/apache/spark/QueryContext.java
b/common/utils/src/main/java/org/apache/spark/QueryContext.java
similarity index 100%
rename from core/src/main/java/org/apache/spark/QueryContext.java
rename to common/utils/src/main/java/org/apache/spark/QueryContext.java
diff --git a/core/src/main/java/org/apache/spark/SparkThrowable.java
b/common/utils/src/main/java/org/apache/spark/SparkThrowable.java
similarity index 100%
rename from core/src/main/java/org/apache/spark/SparkThrowable.java
rename to common/utils/src/main/java/org/apache/spark/SparkThrowable.java
diff --git a/core/src/main/scala/org/apache/spark/ErrorClassesJSONReader.scala
b/common/utils/src/main/scala/org/apache/spark/ErrorClassesJSONReader.scala
similarity index 100%
rename from core/src/main/scala/org/apache/spark/ErrorClassesJSONReader.scala
rename to
common/utils/src/main/scala/org/apache/spark/ErrorClassesJSONReader.scala
diff --git a/core/src/main/scala/org/apache/spark/SparkException.scala
b/common/utils/src/main/scala/org/apache/spark/SparkException.scala
similarity index 95%
rename from core/src/main/scala/org/apache/spark/SparkException.scala
rename to common/utils/src/main/scala/org/apache/spark/SparkException.scala
index 4e48e9c8d41..eb908b826f7 100644
--- a/core/src/main/scala/org/apache/spark/SparkException.scala
+++ b/common/utils/src/main/scala/org/apache/spark/SparkException.scala
@@ -24,8 +24,6 @@ import java.util.ConcurrentModificationException
import scala.collection.JavaConverters._
-import org.apache.hadoop.fs.FileAlreadyExistsException
-
class SparkException(
message: String,
cause: Throwable,
@@ -209,21 +207,6 @@ private[spark] class SparkDateTimeException(
override def getQueryContext: Array[QueryContext] = context
}
-/**
- * Hadoop file already exists exception thrown from Spark with an error class.
- */
-private[spark] class SparkFileAlreadyExistsException(
- errorClass: String,
- messageParameters: Map[String, String])
- extends FileAlreadyExistsException(
- SparkThrowableHelper.getMessage(errorClass, messageParameters))
- with SparkThrowable {
-
- override def getMessageParameters: java.util.Map[String, String] =
messageParameters.asJava
-
- override def getErrorClass: String = errorClass
-}
-
/**
* File not found exception thrown from Spark with an error class.
*/
diff --git a/core/src/main/scala/org/apache/spark/SparkThrowableHelper.scala
b/common/utils/src/main/scala/org/apache/spark/SparkThrowableHelper.scala
similarity index 96%
rename from core/src/main/scala/org/apache/spark/SparkThrowableHelper.scala
rename to
common/utils/src/main/scala/org/apache/spark/SparkThrowableHelper.scala
index e40368eb619..2ca5b81ffb9 100644
--- a/core/src/main/scala/org/apache/spark/SparkThrowableHelper.scala
+++ b/common/utils/src/main/scala/org/apache/spark/SparkThrowableHelper.scala
@@ -21,8 +21,7 @@ import scala.collection.JavaConverters._
import com.fasterxml.jackson.core.util.MinimalPrettyPrinter
-import org.apache.spark.util.JsonProtocol.toJsonString
-import org.apache.spark.util.Utils
+import org.apache.spark.util.JsonUtils.toJsonString
private[spark] object ErrorMessageFormat extends Enumeration {
val PRETTY, MINIMAL, STANDARD = Value
@@ -34,7 +33,7 @@ private[spark] object ErrorMessageFormat extends Enumeration {
*/
private[spark] object SparkThrowableHelper {
val errorReader = new ErrorClassesJsonReader(
- Seq(Utils.getSparkClassLoader.getResource("error/error-classes.json")))
+ Seq(getClass.getClassLoader.getResource("error/error-classes.json")))
def getMessage(
errorClass: String,
diff --git a/common/utils/src/main/scala/org/apache/spark/util/JsonUtils.scala
b/common/utils/src/main/scala/org/apache/spark/util/JsonUtils.scala
new file mode 100644
index 00000000000..58fa5675c25
--- /dev/null
+++ b/common/utils/src/main/scala/org/apache/spark/util/JsonUtils.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import java.io.ByteArrayOutputStream
+import java.nio.charset.StandardCharsets
+
+import com.fasterxml.jackson.core.{JsonEncoding, JsonGenerator}
+import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
+import com.fasterxml.jackson.module.scala.DefaultScalaModule
+
+
+private[spark] object JsonUtils {
+
+ private val mapper = new ObjectMapper().registerModule(DefaultScalaModule)
+ .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+
+ def toJsonString(block: JsonGenerator => Unit): String = {
+ val baos = new ByteArrayOutputStream()
+ val generator = mapper.createGenerator(baos, JsonEncoding.UTF8)
+ block(generator)
+ generator.close()
+ baos.close()
+ new String(baos.toByteArray, StandardCharsets.UTF_8)
+ }
+}
diff --git a/core/pom.xml b/core/pom.xml
index d60938cd4bb..c1bc33c34f9 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -105,6 +105,11 @@
<artifactId>spark-unsafe_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-common-utils_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>javax.activation</groupId>
<artifactId>activation</artifactId>
diff --git
a/core/src/main/scala/org/apache/spark/SparkFileAlreadyExistsException.scala
b/core/src/main/scala/org/apache/spark/SparkFileAlreadyExistsException.scala
new file mode 100644
index 00000000000..532fd04134e
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/SparkFileAlreadyExistsException.scala
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import scala.collection.JavaConverters._
+
+import org.apache.hadoop.fs.FileAlreadyExistsException
+
+/**
+ * Hadoop file already exists exception thrown from Spark with an error class.
+ */
+private[spark] class SparkFileAlreadyExistsException(
+ errorClass: String,
+ messageParameters: Map[String, String])
+ extends FileAlreadyExistsException(
+ SparkThrowableHelper.getMessage(errorClass, messageParameters))
+ with SparkThrowable {
+
+ override def getMessageParameters: java.util.Map[String, String] =
messageParameters.asJava
+
+ override def getErrorClass: String = errorClass
+}
diff --git a/pom.xml b/pom.xml
index 33f5ee00595..c74da8e3ace 100644
--- a/pom.xml
+++ b/pom.xml
@@ -81,6 +81,7 @@
<module>common/network-common</module>
<module>common/network-shuffle</module>
<module>common/unsafe</module>
+ <module>common/utils</module>
<module>common/tags</module>
<module>core</module>
<module>graphx</module>
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index fef29aedfa8..31646537f4d 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -71,6 +71,12 @@ object MimaExcludes {
ProblemFilters.exclude[Problem]("org.sparkproject.spark_core.protobuf.*"),
ProblemFilters.exclude[Problem]("org.apache.spark.status.protobuf.StoreTypes*"),
+ // SPARK-43265: Move Error framework to a common utils module
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.QueryContext"),
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.SparkException"),
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.SparkException$"),
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.SparkThrowable"),
+
(problem: Problem) => problem match {
case MissingClassProblem(cls) =>
!cls.fullName.startsWith("org.sparkproject.jpmml") &&
!cls.fullName.startsWith("org.sparkproject.dmg.pmml")
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 25f5ea4fa85..717f0c1d1cd 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -57,10 +57,11 @@ object BuildCommons {
val connectClient = ProjectRef(buildLocation, "connect-client-jvm")
val allProjects@Seq(
- core, graphx, mllib, mllibLocal, repl, networkCommon, networkShuffle,
launcher, unsafe, tags, sketch, kvstore, _*
+ core, graphx, mllib, mllibLocal, repl, networkCommon, networkShuffle,
launcher, unsafe, tags, sketch, kvstore,
+ commonUtils, _*
) = Seq(
"core", "graphx", "mllib", "mllib-local", "repl", "network-common",
"network-shuffle", "launcher", "unsafe",
- "tags", "sketch", "kvstore"
+ "tags", "sketch", "kvstore", "common-utils"
).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects ++
Seq(connectCommon, connect, connectClient)
val optionallyEnabledProjects@Seq(kubernetes, mesos, yarn,
@@ -403,7 +404,8 @@ object SparkBuild extends PomBuild {
val mimaProjects = allProjects.filterNot { x =>
Seq(
spark, hive, hiveThriftServer, repl, networkCommon, networkShuffle,
networkYarn,
- unsafe, tags, tokenProviderKafka010, sqlKafka010, connectCommon,
connect, connectClient, protobuf
+ unsafe, tags, tokenProviderKafka010, sqlKafka010, connectCommon,
connect, connectClient, protobuf,
+ commonUtils
).contains(x)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
