This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 7cfc40fff6a [SPARK-39468][CORE] Improve `RpcAddress` to add `[]` to
`IPv6` if needed
7cfc40fff6a is described below
commit 7cfc40fff6a2bb4025b29d8dd9eb66734030a901
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Tue Jun 14 20:00:05 2022 -0700
[SPARK-39468][CORE] Improve `RpcAddress` to add `[]` to `IPv6` if needed
### What changes were proposed in this pull request?
This PR aims to extend the `IPv6` support in `RpcAddress` additionally when
the input doesn't have `[]` properly.
### Why are the changes needed?
Note that Apache Spark already depends on `java.net.URI` `getHost` and
`getPort` and it assumpts `[]`-style IPv6. This PR additionally handles the
case where the given host string doesn't have `[]`.
- `RpcAddress.fromURIString`
https://github.com/apache/spark/blob/683179c6813dbdccebd4063c3aac520020765692/core/src/main/scala/org/apache/spark/rpc/RpcAddress.scala#L40-L43
- `Utils.extractHostPortFromSparkUrl`
https://github.com/apache/spark/blob/683179c6813dbdccebd4063c3aac520020765692/core/src/main/scala/org/apache/spark/util/Utils.scala#L2520-L2524
We need to handle Java URI IPv6 style additionally.
```
jshell> var uri = new java.net.URI("https://[::1]:80";)
uri ==> https://[::1]:80
jshell> uri.getHost()
$4 ==> "[::1]"
jshell> uri.getPort()
$5 ==> 80
```
### Does this PR introduce _any_ user-facing change?
No. This is `private[spark]` class.
### How was this patch tested?
Pass the CIs with newly added test cases.
This is also tested manually on IPv6-only environment with the following
command.
```
$ SERIAL_SBT_TESTS=1 SPARK_LOCAL_HOSTNAME='[2600:.(omitted)..:60cd]'
build/sbt "core/test" -Djava.net.preferIPv6Addresses=true
-Dtest.exclude.tags=org.apache.spark.tags.ExtendedLevelDBTest
...
[info] Run completed in 18 minutes, 43 seconds.
[info] Total number of tests run: 2950
[info] Suites: completed 284, aborted 0
[info] Tests: succeeded 2950, failed 0, canceled 4, ignored 8, pending 0
[info] All tests passed.
[info] Passed: Total 3214, Failed 0, Errors 0, Passed 3214, Ignored 8,
Canceled 4
[success] Total time: 1189 s (19:49), completed Jun 14, 2022, 4:45:55 PM
```
Closes #36868 from dongjoon-hyun/SPARK-39468.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../main/scala/org/apache/spark/rpc/RpcAddress.scala | 4 +++-
core/src/main/scala/org/apache/spark/util/Utils.scala | 2 +-
.../scala/org/apache/spark/rpc/RpcAddressSuite.scala | 18 ++++++++++++++++++
3 files changed, 22 insertions(+), 2 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcAddress.scala
b/core/src/main/scala/org/apache/spark/rpc/RpcAddress.scala
index eb0b26947f5..2a2d2051799 100644
--- a/core/src/main/scala/org/apache/spark/rpc/RpcAddress.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/RpcAddress.scala
@@ -23,7 +23,9 @@ import org.apache.spark.util.Utils
/**
* Address for an RPC environment, with hostname and port.
*/
-private[spark] case class RpcAddress(host: String, port: Int) {
+private[spark] case class RpcAddress(_host: String, port: Int) {
+
+ val host: String = Utils.addBracketsIfNeeded(_host)
def hostPort: String = host + ":" + port
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala
b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 3e4a7e727a8..cf93897f97d 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -1089,7 +1089,7 @@ private[spark] object Utils extends Logging {
addBracketsIfNeeded(customHostname.getOrElse(InetAddresses.toUriString(localIpAddress)))
}
- private def addBracketsIfNeeded(addr: String): String = {
+ private[spark] def addBracketsIfNeeded(addr: String): String = {
if (addr.contains(":") && !addr.contains("[")) {
"[" + addr + "]"
} else {
diff --git a/core/src/test/scala/org/apache/spark/rpc/RpcAddressSuite.scala
b/core/src/test/scala/org/apache/spark/rpc/RpcAddressSuite.scala
index b3223ec61bf..0f7c9d71330 100644
--- a/core/src/test/scala/org/apache/spark/rpc/RpcAddressSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rpc/RpcAddressSuite.scala
@@ -52,4 +52,22 @@ class RpcAddressSuite extends SparkFunSuite {
val address = RpcAddress("1.2.3.4", 1234)
assert(address.toSparkURL == "spark://1.2.3.4:1234")
}
+
+ test("SPARK-39468: IPv6 hostPort") {
+ val address = RpcAddress("::1", 1234)
+ assert(address.host == "[::1]")
+ assert(address.port == 1234)
+ assert(address.hostPort == "[::1]:1234")
+ }
+
+ test("SPARK-39468: IPv6 fromSparkURL") {
+ val address = RpcAddress.fromSparkURL("spark://[::1]:1234")
+ assert(address.host == "[::1]")
+ assert(address.port == 1234)
+ }
+
+ test("SPARK-39468: IPv6 toSparkURL") {
+ val address = RpcAddress("::1", 1234)
+ assert(address.toSparkURL == "spark://[::1]:1234")
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
