This is an automated email from the ASF dual-hosted git repository.
nicholasjiang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new f28aac627 [CELEBORN-1707] Audit all RESTful api calls and use separate
restAuditFile
f28aac627 is described below
commit f28aac62713b7bc153fe5bd5a8521fb99b6a5b48
Author: Wang, Fei <[email protected]>
AuthorDate: Fri Nov 15 19:37:21 2024 +0800
[CELEBORN-1707] Audit all RESTful api calls and use separate restAuditFile
### What changes were proposed in this pull request?
Audit all RESTful api calls and use separate restAuditFile.
### Why are the changes needed?
Audit all the api calls and use the separate log file to easy check the
audit log.
### Does this PR introduce _any_ user-facing change?
No, this feature has not been released.
### How was this patch tested?
```
build/sbt "clean;celeborn-master/testOnly *ApiV1MasterResourceSuite"
```
<img width="1714" alt="image"
src="https://github.com/user-attachments/assets/7b94fd89-005b-4f48-ab24-cc4ae7f473e5";>
Closes #2895 from turboFei/rest_audit_log4j.
Authored-by: Wang, Fei <[email protected]>
Signed-off-by: SteNicholas <[email protected]>
---
charts/celeborn/templates/configmap.yaml | 20 ++++++++++++++++++++
conf/log4j2.xml.template | 20 ++++++++++++++++++++
...cationAuditLogger.scala => RestAuditLogger.scala} | 4 ++--
.../http/authentication/AuthenticationFilter.scala | 18 +++++++++++++-----
4 files changed, 55 insertions(+), 7 deletions(-)
diff --git a/charts/celeborn/templates/configmap.yaml
b/charts/celeborn/templates/configmap.yaml
index 052f41ce1..caadd3032 100644
--- a/charts/celeborn/templates/configmap.yaml
+++ b/charts/celeborn/templates/configmap.yaml
@@ -100,6 +100,23 @@ data:
</Delete>
</DefaultRolloverStrategy>
</RollingRandomAccessFile>
+ <RollingRandomAccessFile name="restAuditFile"
fileName="${env:CELEBORN_LOG_DIR}/audit/rest-audit.log"
+
filePattern="${env:CELEBORN_LOG_DIR}/audit/rest-audit.log.%d-%i">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss,SSS} %p [%t]
%c{1}: %m%n%ex"/>
+ <Policies>
+ <SizeBasedTriggeringPolicy size="200 MB"/>
+ </Policies>
+ <DefaultRolloverStrategy max="7">
+ <Delete basePath="${env:CELEBORN_LOG_DIR}/audit"
maxDepth="1">
+ <IfFileName glob="rest-audit.log*">
+ <IfAny>
+ <IfAccumulatedFileSize exceeds="1 GB"/>
+ <IfAccumulatedFileCount exceeds="10"/>
+ </IfAny>
+ </IfFileName>
+ </Delete>
+ </DefaultRolloverStrategy>
+ </RollingRandomAccessFile>
</Appenders>
<Loggers>
@@ -111,6 +128,9 @@ data:
<Appender-ref ref="stdout" level="WARN" />
<Appender-ref ref="file" level="WARN"/>
</Logger>
+ <Logger
name="org.apache.celeborn.server.common.http.RestAuditLogger" level="INFO"
additivity="false">
+ <Appender-ref ref="restAuditFile" level="INFO"/>
+ </Logger>
</Loggers>
</Configuration>
diff --git a/conf/log4j2.xml.template b/conf/log4j2.xml.template
index 5f78b4e23..efd1511e4 100644
--- a/conf/log4j2.xml.template
+++ b/conf/log4j2.xml.template
@@ -49,6 +49,23 @@
</Delete>
</DefaultRolloverStrategy>
</RollingRandomAccessFile>
+ <RollingRandomAccessFile name="restAuditFile"
fileName="${env:CELEBORN_LOG_DIR}/audit/rest-audit.log"
+
filePattern="${env:CELEBORN_LOG_DIR}/audit/rest-audit.log.%d-%i">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss,SSS} %p [%t] %c{1}:
%m%n%ex"/>
+ <Policies>
+ <SizeBasedTriggeringPolicy size="200 MB"/>
+ </Policies>
+ <DefaultRolloverStrategy max="7">
+ <Delete basePath="${env:CELEBORN_LOG_DIR}/audit" maxDepth="1">
+ <IfFileName glob="rest-audit.log*">
+ <IfAny>
+ <IfAccumulatedFileSize exceeds="1 GB"/>
+ <IfAccumulatedFileCount exceeds="10"/>
+ </IfAny>
+ </IfFileName>
+ </Delete>
+ </DefaultRolloverStrategy>
+ </RollingRandomAccessFile>
</Appenders>
<Loggers>
@@ -67,5 +84,8 @@
<!-- <Appender-ref ref="stdout" level="WARN"/>-->
<Appender-ref ref="file" level="WARN"/>
</Logger>
+ <Logger name="org.apache.celeborn.server.common.http.RestAuditLogger"
level="INFO" additivity="false">
+ <Appender-ref ref="restAuditFile" level="INFO"/>
+ </Logger>
</Loggers>
</Configuration>
diff --git
a/service/src/main/scala/org/apache/celeborn/server/common/http/authentication/AuthenticationAuditLogger.scala
b/service/src/main/scala/org/apache/celeborn/server/common/http/RestAuditLogger.scala
similarity index 94%
rename from
service/src/main/scala/org/apache/celeborn/server/common/http/authentication/AuthenticationAuditLogger.scala
rename to
service/src/main/scala/org/apache/celeborn/server/common/http/RestAuditLogger.scala
index 614719daf..d63b96090 100644
---
a/service/src/main/scala/org/apache/celeborn/server/common/http/authentication/AuthenticationAuditLogger.scala
+++
b/service/src/main/scala/org/apache/celeborn/server/common/http/RestAuditLogger.scala
@@ -15,14 +15,14 @@
* limitations under the License.
*/
-package org.apache.celeborn.server.common.http.authentication
+package org.apache.celeborn.server.common.http
import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
import org.apache.celeborn.common.internal.Logging
import
org.apache.celeborn.server.common.http.authentication.AuthenticationFilter._
-object AuthenticationAuditLogger extends Logging {
+object RestAuditLogger extends Logging {
final private val AUDIT_BUFFER = new ThreadLocal[StringBuilder]() {
override protected def initialValue: StringBuilder = new StringBuilder()
}
diff --git
a/service/src/main/scala/org/apache/celeborn/server/common/http/authentication/AuthenticationFilter.scala
b/service/src/main/scala/org/apache/celeborn/server/common/http/authentication/AuthenticationFilter.scala
index f2a3a0709..092541195 100644
---
a/service/src/main/scala/org/apache/celeborn/server/common/http/authentication/AuthenticationFilter.scala
+++
b/service/src/main/scala/org/apache/celeborn/server/common/http/authentication/AuthenticationFilter.scala
@@ -31,6 +31,7 @@ import
org.apache.celeborn.common.authentication.HttpAuthSchemes.HttpAuthScheme
import org.apache.celeborn.common.internal.Logging
import org.apache.celeborn.server.common.Service
import
org.apache.celeborn.server.common.http.HttpAuthUtils.AUTHORIZATION_HEADER
+import org.apache.celeborn.server.common.http.RestAuditLogger
class AuthenticationFilter(conf: CelebornConf, serviceName: String) extends
Filter with Logging {
import AuthenticationFilter._
@@ -133,15 +134,22 @@ class AuthenticationFilter(conf: CelebornConf,
serviceName: String) extends Filt
val httpRequest = request.asInstanceOf[HttpServletRequest]
val httpResponse = response.asInstanceOf[HttpServletResponse]
+ HTTP_CLIENT_IP_ADDRESS.set(httpRequest.getRemoteAddr)
+
HTTP_PROXY_HEADER_CLIENT_IP_ADDRESS.set(httpRequest.getHeader(proxyClientIpHeader))
+
if (authSchemeHandlers.isEmpty ||
BYPASS_API_PATHS.contains(httpRequest.getRequestURI)) {
- filterChain.doFilter(request, response)
- return
+ try {
+ filterChain.doFilter(request, response)
+ return
+ } finally {
+ RestAuditLogger.audit(httpRequest, httpResponse)
+ HTTP_CLIENT_IP_ADDRESS.remove()
+ HTTP_PROXY_HEADER_CLIENT_IP_ADDRESS.remove()
+ }
}
val authorization = httpRequest.getHeader(AUTHORIZATION_HEADER)
val matchedHandler = getMatchedHandler(authorization).orNull
- HTTP_CLIENT_IP_ADDRESS.set(httpRequest.getRemoteAddr)
-
HTTP_PROXY_HEADER_CLIENT_IP_ADDRESS.set(httpRequest.getHeader(proxyClientIpHeader))
try {
if (matchedHandler == null) {
@@ -164,7 +172,7 @@ class AuthenticationFilter(conf: CelebornConf, serviceName:
String) extends Filt
HTTP_AUTH_TYPE.remove()
httpResponse.sendError(HttpServletResponse.SC_FORBIDDEN, e.getMessage)
} finally {
- AuthenticationAuditLogger.audit(httpRequest, httpResponse)
+ RestAuditLogger.audit(httpRequest, httpResponse)
}
}
