This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new a99e1a2b5 [KYUUBI #6185] Audit kyuubi operation state change log
a99e1a2b5 is described below
commit a99e1a2b586edf22a01c0d20ce7cf5b8f1a598c7
Author: Fei Wang <[email protected]>
AuthorDate: Fri Mar 22 10:33:53 2024 +0800
[KYUUBI #6185] Audit kyuubi operation state change log
# :mag: Description
## Issue References ๐
We meet some issue, I want to check the operation sessionHandle and the
operation timeline.
I found that, it is difficult for me to check it from kyuubi log.
So, In this pr, I log the operation change log into separate file.
## Describe Your Solution ๐ง
Please include a summary of the change and which issue is fixed. Please
also include relevant motivation and context. List any dependencies that are
required for this change.
## Types of changes :bookmark:
- [ ] Bugfix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
## Test Plan ๐งช
#### Behavior Without This Pull Request :coffin:
#### Behavior With This Pull Request :tada:
#### Related Unit Tests
---
# Checklist ๐
- [x] This patch was not authored or co-authored using [Generative
Tooling](https://www.apache.org/legal/generative-tooling.html)
**Be nice. Be informative.**
Closes #6185 from turboFei/audit_operation.
Closes #6185
38b01921c [Wang, Fei] op audit
Lead-authored-by: Fei Wang <[email protected]>
Co-authored-by: Wang, Fei <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
conf/log4j2.xml.template | 13 ++++++++
.../kyuubi/operation/AbstractOperation.scala | 2 ++
.../kyuubi/operation/OperationAuditLogger.scala | 38 ++++++++++++++++++++++
kyuubi-server/src/test/resources/log4j2-test.xml | 7 ++++
4 files changed, 60 insertions(+)
diff --git a/conf/log4j2.xml.template b/conf/log4j2.xml.template
index c2db4c475..f1ed28b2e 100644
--- a/conf/log4j2.xml.template
+++ b/conf/log4j2.xml.template
@@ -26,6 +26,8 @@
<Property
name="restAuditLogFilePattern">rest-audit-%d{yyyy-MM-dd}-%i.log</Property>
<Property name="k8sAuditLogPath">k8s-audit.log</Property>
<Property
name="k8sAuditLogFilePattern">k8s-audit-%d{yyyy-MM-dd}-%i.log</Property>
+ <Property name="opAuditLogPath">operation-audit.log</Property>
+ <Property
name="opAuditLogFilePattern">operation-audit-%d{yyyy-MM-dd}-%i.log</Property>
</Properties>
<Appenders>
<Console name="stdout" target="SYSTEM_OUT">
@@ -47,6 +49,14 @@
</Policies>
<DefaultRolloverStrategy max="10"/>
</RollingFile>
+ <RollingFile name="opAudit"
fileName="${sys:logDir}/${sys:opAuditLogPath}"
+ filePattern="${sys:opAuditLogFilePattern}">
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %p %c{1}:
%m%n%ex"/>
+ <Policies>
+ <SizeBasedTriggeringPolicy size="51200KB" />
+ </Policies>
+ <DefaultRolloverStrategy max="10"/>
+ </RollingFile>
</Appenders>
<Loggers>
<Root level="INFO">
@@ -69,5 +79,8 @@
<Logger
name="org.apache.kyuubi.engine.KubernetesApplicationAuditLogger"
additivity="false">
<AppenderRef ref="k8sAudit" />
</Logger>
+ <Logger name="org.apache.kyuubi.operation.OperationAuditLogger"
additivity="false">
+ <AppenderRef ref="opAudit" />
+ </Logger>
</Loggers>
</Configuration>
diff --git
a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/AbstractOperation.scala
b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/AbstractOperation.scala
index 05dd7fda9..3681e98f7 100644
---
a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/AbstractOperation.scala
+++
b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/AbstractOperation.scala
@@ -74,6 +74,7 @@ abstract class AbstractOperation(session: Session) extends
Operation with Loggin
override def getOperationLog: Option[OperationLog] = None
+ OperationAuditLogger.audit(this, OperationState.INITIALIZED)
@volatile protected var state: OperationState = INITIALIZED
@volatile protected var startTime: Long = _
@volatile protected var completedTime: Long = _
@@ -126,6 +127,7 @@ abstract class AbstractOperation(session: Session) extends
Operation with Loggin
}
state = newState
lastAccessTime = System.currentTimeMillis()
+ OperationAuditLogger.audit(this, state)
}
protected def isClosedOrCanceled: Boolean = {
diff --git
a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/OperationAuditLogger.scala
b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/OperationAuditLogger.scala
new file mode 100644
index 000000000..2187d2251
--- /dev/null
+++
b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/OperationAuditLogger.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.operation
+
+import org.apache.kyuubi.Logging
+import org.apache.kyuubi.operation.OperationState.OperationState
+
+object OperationAuditLogger extends Logging {
+ final private val AUDIT_BUFFER = new ThreadLocal[StringBuilder]() {
+ override protected def initialValue: StringBuilder = new StringBuilder()
+ }
+
+ def audit(operation: Operation, state: OperationState): Unit = {
+ val sb = AUDIT_BUFFER.get()
+ sb.setLength(0)
+ sb.append(s"operation=${operation.getHandle.identifier}").append("\t")
+ sb.append(s"opType=${operation.getClass.getSimpleName}").append("\t")
+ sb.append(s"state=$state").append("\t")
+ sb.append(s"user=${operation.getSession.user}").append("\t")
+ sb.append(s"session=${operation.getSession.handle.identifier}")
+ info(sb.toString())
+ }
+}
diff --git a/kyuubi-server/src/test/resources/log4j2-test.xml
b/kyuubi-server/src/test/resources/log4j2-test.xml
index bccbf1b0d..343da22bd 100644
--- a/kyuubi-server/src/test/resources/log4j2-test.xml
+++ b/kyuubi-server/src/test/resources/log4j2-test.xml
@@ -21,6 +21,7 @@
<Configuration status="WARN">
<Properties>
<Property name="restAuditLogPath">target/rest-audit.log</Property>
+ <Property name="opAuditLogPath">target/operation-audit.log</Property>
</Properties>
<Appenders>
<Console name="stdout" target="SYSTEM_OUT">
@@ -39,6 +40,9 @@
<File name="restAudit" fileName="${sys:restAuditLogPath}">
<PatternLayout pattern="%d{HH:mm:ss.SSS} %p %c{1}: %m%n%ex"/>
</File>
+ <File name="opAudit" fileName="${sys:opAuditLogPath}">
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} %p %c{1}: %m%n%ex"/>
+ </File>
</Appenders>
<Loggers>
<Root level="INFO">
@@ -48,6 +52,9 @@
<Logger
name="org.apache.kyuubi.server.http.authentication.AuthenticationAuditLogger"
additivity="false">
<AppenderRef ref="restAudit" />
</Logger>
+ <Logger name="org.apache.kyuubi.operation.OperationAuditLogger"
additivity="false">
+ <AppenderRef ref="opAudit" />
+ </Logger>
<Logger name="org.apache.kyuubi.server.metadata.jdbc" level="DEBUG"
additivity="false">
<AppenderRef ref="stdout" />
<AppenderRef ref="file"/>
