yaooqinn commented on a change in pull request #28651:
URL: https://github.com/apache/spark/pull/28651#discussion_r435946737
##########
File path:
sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
##########
@@ -73,11 +68,19 @@ trait SharedThriftServer extends SharedSparkSession {
}
}
- private def startThriftServer(port: Int, attempt: Int): Unit = {
- logInfo(s"Trying to start HiveThriftServer2: port=$port, attempt=$attempt")
+ private def startThriftServer(attempt: Int): Unit = {
+ logInfo(s"Trying to start HiveThriftServer2:, attempt=$attempt")
val sqlContext = spark.newSession().sqlContext
- sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname,
port.toString)
+ // Set the HIVE_SERVER2_THRIFT_PORT to 0, so it could randomly pick any
free port to use.
+ // It's much more robust than set a random port generated by ourselves
ahead
+ sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, "0")
hiveServer2 = HiveThriftServer2.startWithContext(sqlContext)
+ hiveServer2.getServices.asScala.foreach {
+ case t: ThriftCLIService if t.getPortNumber != 0 =>
+ serverPort = t.getPortNumber
Review comment:
I run the tests with http mode locally with
```sql
build/sbt "hive-thriftserver/test-only *HiveThriftHttpServerSuite" -Phive
-Phive-thriftserver -Dsbt.override.build.repos=true -Phive-2.3
```
I check the logs in `target/unit-tests.log`
```java
20/06/05 06:35:55.237 pool-1-thread-1 INFO AbstractService:
Service:ThriftHttpCLIService is started.
20/06/05 06:35:55.237 pool-1-thread-1 INFO AbstractService:
Service:HiveServer2 is started.
20/06/05 06:35:55.326 Thread-17 INFO Server: jetty-9.4.18.v20190429; built:
2019-04-29T20:42:08.989Z; git: e1bc35120a6617ee3df052294e433f3a25ce7097; jvm
1.8.0_251-b08
20/06/05 06:35:55.358 Thread-17 INFO session: DefaultSessionIdManager
workerName=node0
20/06/05 06:35:55.358 Thread-17 INFO session: No SessionScavenger set, using
defaults
20/06/05 06:35:55.359 Thread-17 INFO session: node0 Scavenging every 660000ms
20/06/05 06:35:55.366 Thread-17 INFO ContextHandler: Started
o.e.j.s.ServletContextHandler@23f4b16a{/,null,AVAILABLE}
20/06/05 06:35:55.438 Thread-17 INFO AbstractConnector: Started
ServerConnector@1b76f67f{HTTP/1.1,[http/1.1]}{0.0.0.0:55923}
20/06/05 06:35:55.438 Thread-17 INFO Server: Started @7043ms
20/06/05 06:35:55.438 Thread-17 INFO ThriftCLIService: Started
ThriftHttpCLIService in http mode on port 55923 path=/cliservice/* with 5...500
worker threads
20/06/05 06:35:55.442 pool-1-thread-1 INFO Utils: Supplied authorities:
localhost:0
20/06/05 06:35:55.442 pool-1-thread-1 WARN Utils: ***** JDBC param
deprecation *****
20/06/05 06:35:55.442 pool-1-thread-1 WARN Utils: The use of
hive.server2.transport.mode is deprecated.
20/06/05 06:35:55.442 pool-1-thread-1 WARN Utils: Please use transportMode
like so: jdbc:hive2://<host>:<port>/dbName;transportMode=<transport_mode_value>
20/06/05 06:35:55.442 pool-1-thread-1 WARN Utils: ***** JDBC param
deprecation *****
20/06/05 06:35:55.442 pool-1-thread-1 WARN Utils: The use of
hive.server2.thrift.http.path is deprecated.
20/06/05 06:35:55.442 pool-1-thread-1 WARN Utils: Please use httpPath like
so: jdbc:hive2://<host>:<port>/dbName;httpPath=<http_path_value>
20/06/05 06:35:55.442 pool-1-thread-1 INFO Utils: Resolved authority:
localhost:0
20/06/05 06:35:55.663 pool-1-thread-1 ERROR HiveConnection: Error opening
session
org.apache.thrift.transport.TTransportException:
org.apache.http.conn.HttpHostConnectException: Connect to localhost:80
[localhost/127.0.0.1, localhost/0:0:0:0:0:0:0:1] failed: Connection refused
(Connection refused)
at
org.apache.thrift.transport.THttpClient.flushUsingHttpClient(THttpClient.java:297)
at org.apache.thrift.transport.THttpClient.flush(THttpClient.java:316)
at org.apache.thrift.TServiceClient.sendBase(TServiceClient.java:73)
at org.apache.thrift.TServiceClient.sendBase(TServiceClient.java:62)
at
org.apache.hive.service.rpc.thrift.TCLIService$Client.send_OpenSession(TCLIService.java:162)
at
org.apache.hive.service.rpc.thrift.TCLIService$Client.OpenSession(TCLIService.java:154)
at
org.apache.hive.jdbc.HiveConnection.openSession(HiveConnection.java:680)
at org.apache.hive.jdbc.HiveConnection.<init>(HiveConnection.java:200)
at org.apache.hive.jdbc.HiveDriver.connect(HiveDriver.java:107)
at java.sql.DriverManager.getConnection(DriverManager.java:664)
at java.sql.DriverManager.getConnection(DriverManager.java:247)
at
org.apache.spark.sql.hive.thriftserver.SharedThriftServer.$anonfun$withMultipleConnectionJdbcStatement$1(SharedThriftServer.scala:106)
at
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
at
scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
at
scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)
at scala.collection.TraversableLike.map(TraversableLike.scala:238)
at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
at scala.collection.AbstractTraversable.map(Traversable.scala:108)
at
org.apache.spark.sql.hive.thriftserver.SharedThriftServer.withMultipleConnectionJdbcStatement(SharedThriftServer.scala:106)
at
org.apache.spark.sql.hive.thriftserver.SharedThriftServer.withMultipleConnectionJdbcStatement$(SharedThriftServer.scala:105)
at
org.apache.spark.sql.hive.thriftserver.HiveThriftHttpServerSuite.withMultipleConnectionJdbcStatement(HiveThriftServer2Suites.scala:280)
at
org.apache.spark.sql.hive.thriftserver.SharedThriftServer.withJdbcStatement(SharedThriftServer.scala:141)
at
org.apache.spark.sql.hive.thriftserver.SharedThriftServer.withJdbcStatement$(SharedThriftServer.scala:140)
at
org.apache.spark.sql.hive.thriftserver.HiveThriftHttpServerSuite.withJdbcStatement(HiveThriftServer2Suites.scala:280)
at
org.apache.spark.sql.hive.thriftserver.SharedThriftServer.$anonfun$startThriftServer$4(SharedThriftServer.scala:166)
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at
org.scalatest.concurrent.Eventually.makeAValiantAttempt$1(Eventually.scala:395)
at
org.scalatest.concurrent.Eventually.tryTryAgain$1(Eventually.scala:409)
at org.scalatest.concurrent.Eventually.eventually(Eventually.scala:439)
at org.scalatest.concurrent.Eventually.eventually$(Eventually.scala:391)
at
org.apache.spark.sql.hive.thriftserver.HiveThriftHttpServerSuite.eventually(HiveThriftServer2Suites.scala:280)
at org.scalatest.concurrent.Eventually.eventually(Eventually.scala:308)
at org.scalatest.concurrent.Eventually.eventually$(Eventually.scala:307)
at
org.apache.spark.sql.hive.thriftserver.HiveThriftHttpServerSuite.eventually(HiveThriftServer2Suites.scala:280)
at
org.apache.spark.sql.hive.thriftserver.SharedThriftServer.startThriftServer(SharedThriftServer.scala:165)
at
org.apache.spark.sql.hive.thriftserver.SharedThriftServer.$anonfun$beforeAll$1(SharedThriftServer.scala:71)
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.Try$.apply(Try.scala:213)
at
org.apache.spark.sql.hive.thriftserver.SharedThriftServer.beforeAll(SharedThriftServer.scala:71)
at
org.apache.spark.sql.hive.thriftserver.SharedThriftServer.beforeAll$(SharedThriftServer.scala:68)
at
org.apache.spark.sql.hive.thriftserver.HiveThriftHttpServerSuite.beforeAll(HiveThriftServer2Suites.scala:280)
at
org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:212)
at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:59)
at
org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:317)
at
org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:510)
at sbt.ForkMain$Run$2.call(ForkMain.java:296)
at sbt.ForkMain$Run$2.call(ForkMain.java:286)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.http.conn.HttpHostConnectException: Connect to
localhost:80 [localhost/127.0.0.1, localhost/0:0:0:0:0:0:0:1] failed:
Connection refused (Connection refused)
at
org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:159)
at
org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:373)
at
org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:394)
at
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:237)
at
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at
org.apache.http.impl.execchain.ServiceUnavailableRetryExec.execute(ServiceUnavailableRetryExec.java:85)
at
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:118)
at
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at
org.apache.thrift.transport.THttpClient.flushUsingHttpClient(THttpClient.java:251)
... 53 more
Caused by: java.net.ConnectException: Connection refused (Connection refused)
at java.net.PlainSocketImpl.socketConnect(Native Method)
at
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
at
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
at
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:606)
at
org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:75)
at
org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:142)
... 64 more
20/06/05 06:35:55.665 pool-1-thread-1 WARN HiveConnection: Failed to connect
to localhost:0
```
The starting phase looks ok to me and the port logged assigned `Started
ThriftHttpCLIService in http mode on port 55923 path=/cliservice/* with 5...500`
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
