This is an automated email from the ASF dual-hosted git repository.
nicholasjiang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new d8495e5b6 [CELEBORN-1532][HELM] Read log4j2 and metrics configurations
from file
d8495e5b6 is described below
commit d8495e5b65af09f42ecf7bc55120d1e7963f5f49
Author: Yi Chen <[email protected]>
AuthorDate: Mon Mar 31 11:00:38 2025 +0800
[CELEBORN-1532][HELM] Read log4j2 and metrics configurations from file
### What changes were proposed in this pull request?
- Create file `charts/celeborn/files/conf/log4j2.xml` and
`charts/celeborn/files/conf/metrics.properties` to store log4j2 and metrics
configurations respectively.
- ConfigMap will read configurations from these two files.
### Why are the changes needed?
For better code readability.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Run Helm unit tests locally by:
```
helm unittest charts/celeborn
```
Closes #3182 from ChenYi015/refactor/helm.
Authored-by: Yi Chen <[email protected]>
Signed-off-by: SteNicholas <[email protected]>
---
charts/celeborn/files/conf/log4j2.xml | 85 ++++++++++++++++++++++++++
charts/celeborn/files/conf/metrics.properties | 18 ++++++
charts/celeborn/templates/configmap.yaml | 88 +--------------------------
3 files changed, 106 insertions(+), 85 deletions(-)
diff --git a/charts/celeborn/files/conf/log4j2.xml
b/charts/celeborn/files/conf/log4j2.xml
new file mode 100644
index 000000000..f0860c5ef
--- /dev/null
+++ b/charts/celeborn/files/conf/log4j2.xml
@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<!--
+ ~ Extra logging related to initialization of Log4j.
+ ~ Set to debug or trace if log4j initialization is failing.
+ -->
+<Configuration status="INFO">
+ <Appenders>
+ <Console name="stdout" target="SYSTEM_OUT">
+ <!--
+ ~ In the pattern layout configuration below, we specify an
explicit `%ex` conversion
+ ~ pattern for logging Throwables. If this was omitted, then
(by default) Log4J would
+ ~ implicitly add an `%xEx` conversion pattern which logs
stacktraces with additional
+ ~ class packaging information. That extra information can
sometimes add a substantial
+ ~ performance overhead, so we disable it in our default
logging config.
+ -->
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss,SSS} %p [%t] %c{1}:
%m%n%ex" />
+ </Console>
+ <RollingRandomAccessFile name="file"
fileName="${env:CELEBORN_LOG_DIR}/celeborn.log"
+ filePattern="${env:CELEBORN_LOG_DIR}/celeborn.log.%d-%i">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss,SSS} %p [%t] %c{1}:
%m%n%ex" />
+ <Policies>
+ <SizeBasedTriggeringPolicy size="200 MB" />
+ </Policies>
+ <DefaultRolloverStrategy max="7">
+ <Delete basePath="${env:CELEBORN_LOG_DIR}" maxDepth="1">
+ <IfFileName glob="celeborn.log*">
+ <IfAny>
+ <IfAccumulatedFileSize exceeds="1 GB" />
+ <IfAccumulatedFileCount exceeds="10" />
+ </IfAny>
+ </IfFileName>
+ </Delete>
+ </DefaultRolloverStrategy>
+ </RollingRandomAccessFile>
+ <RollingRandomAccessFile name="restAuditFile"
+ fileName="${env:CELEBORN_LOG_DIR}/audit/rest-audit.log"
+ filePattern="${env:CELEBORN_LOG_DIR}/audit/rest-audit.log.%d-%i">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss,SSS} %p [%t] %c{1}:
%m%n%ex" />
+ <Policies>
+ <SizeBasedTriggeringPolicy size="200 MB" />
+ </Policies>
+ <DefaultRolloverStrategy max="7">
+ <Delete basePath="${env:CELEBORN_LOG_DIR}/audit" maxDepth="1">
+ <IfFileName glob="rest-audit.log*">
+ <IfAny>
+ <IfAccumulatedFileSize exceeds="1 GB" />
+ <IfAccumulatedFileCount exceeds="10" />
+ </IfAny>
+ </IfFileName>
+ </Delete>
+ </DefaultRolloverStrategy>
+ </RollingRandomAccessFile>
+ </Appenders>
+
+ <Loggers>
+ <Root level="INFO">
+ <AppenderRef ref="stdout" />
+ <AppenderRef ref="file" />
+ </Root>
+ <Logger name="org.apache.hadoop.hdfs" level="WARN" additivity="false">
+ <Appender-ref ref="stdout" level="WARN" />
+ <Appender-ref ref="file" level="WARN" />
+ </Logger>
+ <Logger name="org.apache.celeborn.server.common.http.RestAuditLogger"
level="INFO"
+ additivity="false">
+ <Appender-ref ref="restAuditFile" level="INFO" />
+ </Logger>
+ </Loggers>
+</Configuration>
diff --git a/charts/celeborn/files/conf/metrics.properties
b/charts/celeborn/files/conf/metrics.properties
new file mode 100644
index 000000000..7c5630afd
--- /dev/null
+++ b/charts/celeborn/files/conf/metrics.properties
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+*.sink.prometheusServlet.class=org.apache.celeborn.common.metrics.sink.PrometheusServlet
diff --git a/charts/celeborn/templates/configmap.yaml
b/charts/celeborn/templates/configmap.yaml
index caadd3032..5ccaa63d0 100644
--- a/charts/celeborn/templates/configmap.yaml
+++ b/charts/celeborn/templates/configmap.yaml
@@ -50,89 +50,7 @@ data:
{{- end}}
log4j2.xml: |-
- <?xml version="1.0" encoding="UTF-8"?>
- <!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
- <!--
- ~ Extra logging related to initialization of Log4j.
- ~ Set to debug or trace if log4j initialization is failing.
- -->
- <Configuration status="INFO">
- <Appenders>
- <Console name="stdout" target="SYSTEM_OUT">
- <!--
- ~ In the pattern layout configuration below, we specify an
explicit `%ex` conversion
- ~ pattern for logging Throwables. If this was omitted, then
(by default) Log4J would
- ~ implicitly add an `%xEx` conversion pattern which logs
stacktraces with additional
- ~ class packaging information. That extra information can
sometimes add a substantial
- ~ performance overhead, so we disable it in our default
logging config.
- -->
- <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss,SSS} %p [%t]
%c{1}: %m%n%ex"/>
- </Console>
- <RollingRandomAccessFile name="file"
fileName="${env:CELEBORN_LOG_DIR}/celeborn.log"
-
filePattern="${env:CELEBORN_LOG_DIR}/celeborn.log.%d-%i">
- <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss,SSS} %p [%t]
%c{1}: %m%n%ex"/>
- <Policies>
- <SizeBasedTriggeringPolicy size="200 MB"/>
- </Policies>
- <DefaultRolloverStrategy max="7">
- <Delete basePath="${env:CELEBORN_LOG_DIR}" maxDepth="1">
- <IfFileName glob="celeborn.log*">
- <IfAny>
- <IfAccumulatedFileSize exceeds="1 GB" />
- <IfAccumulatedFileCount exceeds="10" />
- </IfAny>
- </IfFileName>
- </Delete>
- </DefaultRolloverStrategy>
- </RollingRandomAccessFile>
- <RollingRandomAccessFile name="restAuditFile"
fileName="${env:CELEBORN_LOG_DIR}/audit/rest-audit.log"
-
filePattern="${env:CELEBORN_LOG_DIR}/audit/rest-audit.log.%d-%i">
- <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss,SSS} %p [%t]
%c{1}: %m%n%ex"/>
- <Policies>
- <SizeBasedTriggeringPolicy size="200 MB"/>
- </Policies>
- <DefaultRolloverStrategy max="7">
- <Delete basePath="${env:CELEBORN_LOG_DIR}/audit"
maxDepth="1">
- <IfFileName glob="rest-audit.log*">
- <IfAny>
- <IfAccumulatedFileSize exceeds="1 GB"/>
- <IfAccumulatedFileCount exceeds="10"/>
- </IfAny>
- </IfFileName>
- </Delete>
- </DefaultRolloverStrategy>
- </RollingRandomAccessFile>
- </Appenders>
+ {{- .Files.Get "files/conf/log4j2.xml" | nindent 4 }}
- <Loggers>
- <Root level="INFO">
- <AppenderRef ref="stdout"/>
- <AppenderRef ref="file"/>
- </Root>
- <Logger name="org.apache.hadoop.hdfs" level="WARN"
additivity="false">
- <Appender-ref ref="stdout" level="WARN" />
- <Appender-ref ref="file" level="WARN"/>
- </Logger>
- <Logger
name="org.apache.celeborn.server.common.http.RestAuditLogger" level="INFO"
additivity="false">
- <Appender-ref ref="restAuditFile" level="INFO"/>
- </Logger>
- </Loggers>
- </Configuration>
-
- metrics.properties: >-
-
*.sink.prometheusServlet.class=org.apache.celeborn.common.metrics.sink.PrometheusServlet
+ metrics.properties: |-
+ {{- .Files.Get "files/conf/metrics.properties" | nindent 4 }}
