This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new c9273eb [KYUUBI #1050] Add KyuubiServerInfoEvent
c9273eb is described below
commit c9273eb3815c57c818641bba517408e4f8748629
Author: zhenjiaguo <[email protected]>
AuthorDate: Thu Nov 25 13:49:02 2021 +0800
[KYUUBI #1050] Add KyuubiServerInfoEvent
<!--
Thanks for sending a pull request!
Here are some tips for you:
1. If this is your first time, please read our contributor guidelines:
https://kyuubi.readthedocs.io/en/latest/community/contributions.html
2. If the PR is related to an issue in
https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your
PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g.,
'[WIP][KYUUBI #XXXX] Your PR title ...'.
-->
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
Add KyuubiServerStartEvent in KyuubiServer so we can get the server level
info:
- server config
- server version
- server environment
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run
test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #1378 from zhenjiaguo/add-kyuubi-server-event.
Closes #1050
d97e1d00 [zhenjiaguo] optimize import
56939c13 [zhenjiaguo] change import way
7632363b [zhenjiaguo] rebase master
44559154 [zhenjiaguo] add more info in kyuubi server event
beacc88a [zhenjiaguo] add sparksql case-sensitive setting
b35875fc [zhenjiaguo] add kyuubi server start event test
d3161681 [zhenjiaguo] add KyuubiServerEvent
Authored-by: zhenjiaguo <[email protected]>
Signed-off-by: ulysses-you <[email protected]>
---
.../kyuubi/events/KyuubiServerInfoEvent.scala | 79 ++++++++++++++++++++++
.../org/apache/kyuubi/server/KyuubiServer.scala | 9 ++-
.../kyuubi/events/EventLoggingServiceSuite.scala | 65 +++++++++++++++++-
3 files changed, 151 insertions(+), 2 deletions(-)
diff --git
a/kyuubi-server/src/main/scala/org/apache/kyuubi/events/KyuubiServerInfoEvent.scala
b/kyuubi-server/src/main/scala/org/apache/kyuubi/events/KyuubiServerInfoEvent.scala
new file mode 100644
index 0000000..18af357
--- /dev/null
+++
b/kyuubi-server/src/main/scala/org/apache/kyuubi/events/KyuubiServerInfoEvent.scala
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.events
+
+import org.apache.kyuubi.{BUILD_DATE => P_BUILD_DATE, BUILD_USER =>
P_BUILD_USER, REPO_URL => P_REPO_URL, _}
+import org.apache.kyuubi.server.KyuubiServer
+import org.apache.kyuubi.service.ServiceState
+
+/**
+ * A [[KyuubiServerInfoEvent]] is used to get server level info when
KyuubiServer start or stop
+ *
+ * @param serverName the server name
+ * @param startTime the time when the server started
+ * @param eventTime the time when the event made
+ * @param state the server states
+ * @param serverIP the server ip
+ * @param serverConf the server config
+ * @param serverEnv the server environment
+ */
+case class KyuubiServerInfoEvent private (
+ serverName: String,
+ startTime: Long,
+ eventTime: Long,
+ state: String,
+ serverIP: String,
+ serverConf: Map[String, String],
+ serverEnv: Map[String, String]) extends KyuubiServerEvent {
+
+ val BUILD_USER: String = P_BUILD_USER
+ val BUILD_DATE: String = P_BUILD_DATE
+ val REPO_URL: String = P_REPO_URL
+
+ val VERSION_INFO = Map(
+ "KYUUBI_VERSION" -> KYUUBI_VERSION,
+ "JAVA_COMPILE_VERSION" -> JAVA_COMPILE_VERSION,
+ "SCALA_COMPILE_VERSION" -> SCALA_COMPILE_VERSION,
+ "SPARK_COMPILE_VERSION" -> SPARK_COMPILE_VERSION,
+ "HIVE_COMPILE_VERSION" -> HIVE_COMPILE_VERSION,
+ "HADOOP_COMPILE_VERSION" -> HADOOP_COMPILE_VERSION)
+
+ override def partitions: Seq[(String, String)] =
+ ("day", Utils.getDateFromTimestamp(startTime)) :: Nil
+}
+
+object KyuubiServerInfoEvent {
+
+ def apply(
+ server: KyuubiServer,
+ state: ServiceState.ServiceState): Option[KyuubiServerInfoEvent] = {
+ server.getServiceState match {
+ // Only server is started that we can log event
+ case ServiceState.STARTED =>
+ Some(KyuubiServerInfoEvent(
+ server.getName,
+ server.getStartTime,
+ System.currentTimeMillis(),
+ state.toString,
+ server.frontendServices.head.connectionUrl,
+ server.getConf.getAll,
+ server.getConf.getEnvs))
+ case _ => None
+ }
+ }
+}
diff --git
a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
index c194a09..2d200c4 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
@@ -31,11 +31,12 @@ import org.apache.kyuubi._
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.{FRONTEND_PROTOCOLS,
FrontendProtocols}
import org.apache.kyuubi.config.KyuubiConf.FrontendProtocols._
+import org.apache.kyuubi.events.KyuubiServerInfoEvent
import org.apache.kyuubi.ha.HighAvailabilityConf._
import org.apache.kyuubi.ha.client.{ServiceDiscovery, ZooKeeperAuthTypes}
import org.apache.kyuubi.ha.client.ZooKeeperClientProvider._
import org.apache.kyuubi.metrics.{MetricsConf, MetricsSystem}
-import org.apache.kyuubi.service.{AbstractBackendService,
AbstractFrontendService, Serverable}
+import org.apache.kyuubi.service.{AbstractBackendService,
AbstractFrontendService, Serverable, ServiceState}
import org.apache.kyuubi.util.{KyuubiHadoopUtils, SignalRegister}
import org.apache.kyuubi.zookeeper.EmbeddedZookeeper
@@ -157,6 +158,12 @@ class KyuubiServer(name: String) extends Serverable(name) {
override def start(): Unit = {
super.start()
KyuubiServer.kyuubiServer = this
+ KyuubiServerInfoEvent(this,
ServiceState.STARTED).foreach(EventLoggingService.onEvent)
+ }
+
+ override def stop(): Unit = {
+ KyuubiServerInfoEvent(this,
ServiceState.STOPPED).foreach(EventLoggingService.onEvent)
+ super.stop()
}
override protected def stopServer(): Unit = {}
diff --git
a/kyuubi-server/src/test/scala/org/apache/kyuubi/events/EventLoggingServiceSuite.scala
b/kyuubi-server/src/test/scala/org/apache/kyuubi/events/EventLoggingServiceSuite.scala
index b2fb9a7..f64e3ee 100644
---
a/kyuubi-server/src/test/scala/org/apache/kyuubi/events/EventLoggingServiceSuite.scala
+++
b/kyuubi-server/src/test/scala/org/apache/kyuubi/events/EventLoggingServiceSuite.scala
@@ -21,13 +21,16 @@ import java.net.InetAddress
import java.nio.file.Paths
import java.util.UUID
+import com.fasterxml.jackson.databind.ObjectMapper
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path}
-import org.apache.kyuubi.{Utils, WithKyuubiServer}
+import org.apache.kyuubi._
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.operation.HiveJDBCTestHelper
import org.apache.kyuubi.operation.OperationState._
+import org.apache.kyuubi.server.KyuubiServer
+import org.apache.kyuubi.service.ServiceState
class EventLoggingServiceSuite extends WithKyuubiServer with
HiveJDBCTestHelper {
@@ -151,4 +154,64 @@ class EventLoggingServiceSuite extends WithKyuubiServer
with HiveJDBCTestHelper
}
}
}
+
+ test("test Kyuubi server info event") {
+ val confKv = List(("awesome.kyuubi", "true"), ("awesome.kyuubi.server",
"yeah"))
+ confKv.foreach(kv => conf.set(kv._1, kv._2))
+
+ val name = "KyuubiServerInfoTest"
+ val server = new KyuubiServer(name)
+ server.initialize(conf)
+ server.start()
+ server.stop()
+
+ val hostName = InetAddress.getLocalHost.getCanonicalHostName
+ val kyuubiServerInfoPath =
+ Paths.get(serverLogRoot, "kyuubi_server_info", s"day=$currentDate",
s"server-$hostName.json")
+
+ withJdbcStatement() { statement =>
+ statement.executeQuery("set spark.sql.caseSensitive=true")
+ val res = statement.executeQuery(
+ s"SELECT * FROM `json`.`$kyuubiServerInfoPath` where serverName =
'$name'")
+
+ res.next()
+ assert(res.getString("serverName") == name)
+ assert(res.getLong("startTime") > 0)
+
+ val startEventTime = res.getLong("eventTime")
+ assert(res.getLong("startTime") <= startEventTime)
+ assert(res.getString("state") == ServiceState.STARTED.toString)
+
+ val serverIP = res.getString("serverIP")
+ assert(serverIP != null && !"".equals(serverIP))
+
+ val objMapper = new ObjectMapper()
+ val confMap = objMapper.readTree(res.getString("serverConf"))
+ confKv.foreach(kv => assert(confMap.get(kv._1).asText() == kv._2))
+
+ val USER = "USER"
+ val envMap = objMapper.readTree(res.getString("serverEnv"))
+ val envUser = if (envMap.has(USER)) envMap.get(USER).asText() else ""
+ assert(envUser == sys.env.getOrElse(USER, ""))
+
+ assert(res.getString("BUILD_USER") == BUILD_USER)
+ assert(res.getString("BUILD_DATE") == BUILD_DATE)
+ assert(res.getString("REPO_URL") == REPO_URL)
+
+ val versionInfoMap = objMapper.readTree(res.getString("VERSION_INFO"))
+ assert(versionInfoMap.get("KYUUBI_VERSION").asText() == KYUUBI_VERSION)
+ assert(versionInfoMap.get("JAVA_COMPILE_VERSION").asText() ==
JAVA_COMPILE_VERSION)
+ assert(versionInfoMap.get("SCALA_COMPILE_VERSION").asText() ==
SCALA_COMPILE_VERSION)
+ assert(versionInfoMap.get("HIVE_COMPILE_VERSION").asText() ==
HIVE_COMPILE_VERSION)
+ assert(versionInfoMap.get("HADOOP_COMPILE_VERSION").asText() ==
HADOOP_COMPILE_VERSION)
+ assert(res.getString("eventType") == "kyuubi_server_info")
+
+ res.next()
+ assert(res.getString("serverName") == name)
+ assert(startEventTime <= res.getLong("eventTime"))
+ assert(res.getString("state") == ServiceState.STOPPED.toString)
+ assert(res.getString("BUILD_USER") == BUILD_USER)
+ assert(res.getString("eventType") == "kyuubi_server_info")
+ }
+ }
}
