This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 103969a [KYUUBI #1067] Catch the real exception when the log
directory is not exist
103969a is described below
commit 103969a04216a812d41beb32dfbac75abb12e0ad
Author: 张宇翔 <[email protected]>
AuthorDate: Fri Sep 24 20:24:20 2021 +0800
[KYUUBI #1067] Catch the real exception when the log directory is not exist
BUGFIX:
#1067
fix:
2021-09-23 16:03:43.248 INFO HiveMetaStore.audit: ugi=zhangyu
ip=unknown-ip-addr cmd=get_databases: *
2021-09-23 16:03:43.319 ERROR events.JsonEventLogger: Get log directory
/Users/zhangyuxiang/Downloads/kyuubi_test failed
java.io.FileNotFoundException: File
/Users/zhangyuxiang/Downloads/kyuubi_test does not exist
at
org.apache.hadoop.fs.RawLocalFileSystem.deprecatedGetFileStatus(RawLocalFileSystem.java:666)
at
org.apache.hadoop.fs.RawLocalFileSystem.getFileLinkStatusInternal(RawLocalFileSystem.java:987)
at
org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:656)
at
org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:454)
at
org.apache.kyuubi.events.JsonEventLogger.requireLogRootWritable(JsonEventLogger.scala:79)
at
org.apache.kyuubi.events.JsonEventLogger.initialize(JsonEventLogger.scala:93)
at
org.apache.kyuubi.service.CompositeService.$anonfun$initialize$1(CompositeService.scala:40)
at
org.apache.kyuubi.service.CompositeService.$anonfun$initialize$1$adapted(CompositeService.scala:40)
at
scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at
scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run
test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #1136 from zhang1002/branch-1.3.0_checkRightException.
Closes #1067
de16c63d [张宇翔] delete some ut
5a29bb80 [张宇翔] Merge branch 'master' into branch-1.3.0_checkRightException
21e856a3 [张宇翔] Merge branch 'master' into branch-1.3.0_checkRightException
873cc9c4 [张宇翔] Merge remote-tracking branch 'upstream/master'
d1e7a019 [张宇翔] catch IOException when log directory is not exist
bc442d8c [张宇翔] catch IOException when log directory is not exist
0d7418e9 [张宇翔] Merge branch 'master' into branch-1.3.0_checkRightException
41cff19a [张宇翔] Merge remote-tracking branch 'upstream/master'
74b36efb [张宇翔] catch IOException when log directory is not exist
def8d1f6 [张宇翔] Merge remote-tracking branch 'upstream/master'
8384a6e2 [张宇翔] Merge remote-tracking branch 'upstream/master'
714ab1b8 [张宇翔] Merge remote-tracking branch 'upstream/master'
af28520b [张宇翔] Merge remote-tracking branch 'upstream/master'
c328c884 [张宇翔] Merge remote-tracking branch 'upstream/master'
abcaae1f [张宇翔] Merge remote-tracking branch 'upstream/master'
f0a7e566 [张宇翔] Merge remote-tracking branch 'upstream/master'
112aa6b6 [张宇翔] Merge remote-tracking branch 'upstream/master'
c55652fd [张宇翔] Merge remote-tracking branch 'upstream/master'
ef5e8c55 [张宇翔] Merge remote-tracking branch 'upstream/master'
5a75384c [张宇翔] Merge remote-tracking branch 'upstream/master'
8da4f7ed [张宇翔] Merge remote-tracking branch 'upstream/master'
a7ba1bfc [张宇翔] Merge remote-tracking branch 'upstream/master'
b662a989 [张宇翔] Merge remote-tracking branch 'upstream/master'
4c8f3b87 [张宇翔] Merge remote-tracking branch 'upstream/master'
8b686767 [张宇翔] Merge remote-tracking branch 'upstream/master'
cf99e309 [张宇翔] Merge remote-tracking branch 'upstream/master'
0afaa578 [张宇翔] Merge remote-tracking branch 'upstream/master'
b24fea07 [张宇翔] Merge remote-tracking branch 'upstream/master'
e517cfc5 [张宇翔] Merge remote-tracking branch 'upstream/master'
18aebe76 [张宇翔] Merge remote-tracking branch 'upstream/master'
f248bef7 [张宇翔] Merge remote-tracking branch 'upstream/master'
5ffb54f3 [张宇翔] Add kyuubi-spark-monitor module for nightly.yml
Authored-by: 张宇翔 <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
---
.../scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala | 10 +++++++---
.../org/apache/kyuubi/engine/spark/events/EngineEvent.scala | 7 ++++++-
2 files changed, 13 insertions(+), 4 deletions(-)
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala
index afed700..a25b3db 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala
@@ -61,8 +61,10 @@ case class SparkSQLEngine(spark: SparkSession) extends
Serverable("SparkSQLEngin
}
override def stop(): Unit = {
- eventLogging.onEvent(
- engineStatus.copy(state = ServiceState.STOPPED.id, endTime =
System.currentTimeMillis()))
+ if (!state.equals(ServiceState.LATENT)) {
+ eventLogging.onEvent(
+ engineStatus.copy(state = ServiceState.STOPPED.id, endTime =
System.currentTimeMillis()))
+ }
super.stop()
}
@@ -147,7 +149,9 @@ object SparkSQLEngine extends Logging {
currentEngine.foreach { engine =>
val status =
engine.engineStatus.copy(diagnostic = s"Error State SparkSQL
Engine ${t.getMessage}")
- EventLoggingService.onEvent(status)
+ if (!engine.getServiceState.equals(ServiceState.LATENT)) {
+ EventLoggingService.onEvent(status)
+ }
error(status, t)
engine.stop()
}
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/EngineEvent.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/EngineEvent.scala
index eee8943..b9245b0 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/EngineEvent.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/EngineEvent.scala
@@ -98,13 +98,18 @@ object EngineEvent {
val webUrl = sc.getConf.getOption(
"spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_URI_BASES")
.orElse(sc.uiWebUrl).getOrElse("")
+ val connectionUrl = if
(engine.getServiceState.equals(ServiceState.LATENT)) {
+ null
+ } else {
+ engine.frontendServices.head.connectionUrl
+ }
new EngineEvent(
sc.applicationId,
sc.applicationAttemptId,
sc.appName,
sc.sparkUser,
engine.getConf.get(ENGINE_SHARE_LEVEL),
- engine.frontendServices.head.connectionUrl,
+ connectionUrl,
sc.master,
sc.version,
webUrl,
