Github user heary-cao commented on a diff in the pull request:
https://github.com/apache/spark/pull/20516#discussion_r166814031
--- Diff:
core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferServiceSuite.scala
---
@@ -77,16 +79,53 @@ class NettyBlockTransferServiceSuite
verifyServicePort(expectedPort = service0.port + 1, actualPort =
service1.port)
}
+ test("can bind to two max specific ports") {
+ service0 = createService(port = 65535)
+ service1 = createService(port = 65535)
+ verifyServicePort(expectedPort = 65535, actualPort = service0.port)
+ // see `Utils.userPort` the user port to try when trying to bind a
service,
+ // the max privileged port is 1024.
+ verifyServicePort(expectedPort = 1024, actualPort = service1.port)
+ }
+
+ test("can't bind to a privileged port") {
+ intercept[IllegalArgumentException] {
+ service0 = createService(port = 23)
+ }
+ }
+
+ test("turn off spark.port.maxRetries, bind repeat port is fail") {
+ val port = 17634 + Random.nextInt(10000)
+ logInfo("random port for test: " + port)
+ service0 = createService(port)
+
+ // `service0.port` is occupied, bind repeat port throw BindException.
+ intercept[BindException] {
+ val conf = new SparkConf()
+ .set("spark.app.id", s"test-${getClass.getName}")
+ .set("spark.testing", "true")
+ .set("spark.port.maxRetries", "0")
+
+ val securityManager = new SecurityManager(conf)
+ val blockDataManager = mock(classOf[BlockDataManager])
+ val service = new NettyBlockTransferService(conf, securityManager,
"localhost", "localhost",
+ service0.port, 1)
+ service.init(blockDataManager)
+ }
+ }
+
private def verifyServicePort(expectedPort: Int, actualPort: Int): Unit
= {
actualPort should be >= expectedPort
// avoid testing equality in case of simultaneous tests
+ // `spark.testing` is true,
// the default value for `spark.port.maxRetries` is 100 under test
actualPort should be <= (expectedPort + 100)
}
private def createService(port: Int): NettyBlockTransferService = {
val conf = new SparkConf()
.set("spark.app.id", s"test-${getClass.getName}")
+ .set("spark.testing", "true")
--- End diff --
@srowen,
if you don't set the spark.testing for true, the default value for
spark.port.maxRetries is not 100, but 16,
so in verifyServicePort function,
actualPort should be <= (expectedPort + 100) is not true.
because
def portMaxRetries(conf: SparkConf): Int = {
val maxRetries = conf.getOption("spark.port.maxRetries").map(_.toInt)
if (conf.contains("spark.testing")) {
// Set a higher number of retries for tests...
maxRetries.getOrElse(100)
} else {
maxRetries.getOrElse(16)
}
}
thanks.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
