This is an automated email from the ASF dual-hosted git repository.
kabhwan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6a754b1344d [SPARK-40657][PROTOBUF][FOLLOWUP][MINOR] Add clarifying
comment in ProtobufUtils
6a754b1344d is described below
commit 6a754b1344d660c25aebf586995a1354c9207d7a
Author: Raghu Angadi <raghu.ang...@databricks.com>
AuthorDate: Fri Nov 18 09:41:08 2022 +0900
[SPARK-40657][PROTOBUF][FOLLOWUP][MINOR] Add clarifying comment in
ProtobufUtils
### What changes were proposed in this pull request?
This is a follow up to address couple of comments in #38384.
Fixes a comment and adds explanation about why we don't use
"com.google.protobuf.Message" directly in a string.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing unit tests
Closes #38680 from rangadi/minor-fix.
Authored-by: Raghu Angadi <raghu.ang...@databricks.com>
Signed-off-by: Jungtaek Lim <kabhwan.opensou...@gmail.com>
---
.../org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala | 6 ++++--
.../sql/protobuf/ProtobufCatalystDataConversionSuite.scala | 10 +++++-----
2 files changed, 9 insertions(+), 7 deletions(-)
diff --git
a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala
b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala
index b91ffbd858a..0ecb728060d 100644
---
a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala
+++
b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala
@@ -172,7 +172,7 @@ private[sql] object ProtobufUtils extends Logging {
throw QueryCompilationErrors.protobufClassLoadError(protobufClassName,
explanation, e)
case e: NoClassDefFoundError if
e.getMessage.matches("com/google/proto.*Generated.*") =>
- // This indicates the the the Java classes are not shaded.
+ // This indicates the Java classes are not shaded.
throw QueryCompilationErrors.protobufClassLoadError(
protobufClassName, missingShadingErrorMessage, e)
}
@@ -180,7 +180,9 @@ private[sql] object ProtobufUtils extends Logging {
if (!shadedMessageClass.isAssignableFrom(protobufClass)) {
// Check if this extends 2.x Message class included in spark, that does
not work.
val unshadedMessageClass = Utils.classForName(
-
"com.escape-shading.google.protobuf.Message".replace("escape-shading.", "")
+ // Generate "com.google.protobuf.Message". Using join() is a trick to
escape from
+ // jar shader. Otherwise, it will be replaced with
'org.sparkproject...'.
+ String.join(".", "com", "google", "protobuf", "Message")
)
val explanation =
if (unshadedMessageClass.isAssignableFrom(protobufClass)) {
diff --git
a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufCatalystDataConversionSuite.scala
b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufCatalystDataConversionSuite.scala
index 9f9b51006ca..ec75ebb5507 100644
---
a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufCatalystDataConversionSuite.scala
+++
b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufCatalystDataConversionSuite.scala
@@ -116,11 +116,11 @@ class ProtobufCatalystDataConversionSuite
StructType(StructField("string_type", StringType, nullable = true) :: Nil))
private val catalystTypesToProtoMessages: Map[DataType, (String, Any)] = Map(
- IntegerType -> ("IntegerMsg", 0),
- DoubleType -> ("DoubleMsg", 0.0d),
- FloatType -> ("FloatMsg", 0.0f),
- BinaryType -> ("BytesMsg", ByteString.empty().toByteArray),
- StringType -> ("StringMsg", ""))
+ (IntegerType, ("IntegerMsg", 0)), // Don't use '->', it causes a scala
warning.
+ (DoubleType, ("DoubleMsg", 0.0d)),
+ (FloatType, ("FloatMsg", 0.0f)),
+ (BinaryType, ("BytesMsg", ByteString.empty().toByteArray)),
+ (StringType, ("StringMsg", "")))
testingTypes.foreach { dt =>
val seed = scala.util.Random.nextInt(RandomDataGenerator.MAX_STR_LEN)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
