This is an automated email from the ASF dual-hosted git repository.
nicholasjiang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 3d61cede3 [CELEBORN-2207] Master StatefulSet should add
ratis-log4j.properties for ratis shell
3d61cede3 is described below
commit 3d61cede3f42155e3f895c52acc30e069655b77c
Author: SteNicholas <[email protected]>
AuthorDate: Tue Nov 18 14:54:49 2025 +0800
[CELEBORN-2207] Master StatefulSet should add ratis-log4j.properties for
ratis shell
### What changes were proposed in this pull request?
Master StatefulSet should add `ratis-log4j.properties` for ratis shell.
### Why are the changes needed?
Master StatefulSet does not include `ratis-log4j.properties`, which cause
that `${CELEBORN_CONF_DIR}/ratis-log4j.properties` not exists for ratis shell.
Therefore, StatefulSet should add r`atis-log4j.properties` for ratis shell of
master.
### Does this PR resolve a correctness bug?
No.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
`Test Celeborn configmap#Should have correct data fields`
Closes #3542 from SteNicholas/CELEBORN-2207.
Authored-by: SteNicholas <[email protected]>
Signed-off-by: SteNicholas <[email protected]>
---
charts/celeborn/files/conf/ratis-log4j.properties | 31 ++++++++++++++++++++++
charts/celeborn/templates/configmap.yaml | 3 +++
charts/celeborn/templates/master/statefulset.yaml | 5 ++++
charts/celeborn/tests/configmap_test.yaml | 4 +++
charts/celeborn/tests/master/statefulset_test.yaml | 6 ++---
charts/celeborn/tests/worker/statefulset_test.yaml | 6 ++---
6 files changed, 49 insertions(+), 6 deletions(-)
diff --git a/charts/celeborn/files/conf/ratis-log4j.properties
b/charts/celeborn/files/conf/ratis-log4j.properties
new file mode 100644
index 000000000..f75a6b1c4
--- /dev/null
+++ b/charts/celeborn/files/conf/ratis-log4j.properties
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# These properties may be overridden by system properties. log4j gives system
properties
+# a higher precedence than locally defined variables.
+log4j.rootLogger=INFO, SHELL_LOGGER
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2}
(%F:%M) - %m%n
+
+# SHELL Logger
+log4j.appender.SHELL_LOGGER=org.apache.log4j.RollingFileAppender
+log4j.appender.SHELL_LOGGER.File=${ratis.shell.logs.dir}/celeborn-ratis-shell.log
+log4j.appender.SHELL_LOGGER.MaxFileSize=100MB
+log4j.appender.SHELL_LOGGER.MaxBackupIndex=10
+log4j.appender.SHELL_LOGGER.layout=org.apache.log4j.PatternLayout
+log4j.appender.SHELL_LOGGER.layout.ConversionPattern=%d{ISO8601} [%t] %-5p
%c{2} (%F:%M) - %m%n
diff --git a/charts/celeborn/templates/configmap.yaml
b/charts/celeborn/templates/configmap.yaml
index e412af41d..7207d5fb0 100644
--- a/charts/celeborn/templates/configmap.yaml
+++ b/charts/celeborn/templates/configmap.yaml
@@ -48,3 +48,6 @@ data:
metrics.properties: |-
{{- .Files.Get "files/conf/metrics.properties" | nindent 4 }}
+
+ ratis-log4j.properties: |-
+ {{- .Files.Get "files/conf/ratis-log4j.properties" | nindent 4 }}
diff --git a/charts/celeborn/templates/master/statefulset.yaml
b/charts/celeborn/templates/master/statefulset.yaml
index e962b2a5e..987d0aa28 100644
--- a/charts/celeborn/templates/master/statefulset.yaml
+++ b/charts/celeborn/templates/master/statefulset.yaml
@@ -122,6 +122,9 @@ spec:
- name: celeborn-conf
subPath: metrics.properties
mountPath: /opt/celeborn/conf/metrics.properties
+ - name: celeborn-conf
+ subPath: ratis-log4j.properties
+ mountPath: /opt/celeborn/conf/ratis-log4j.properties
{{- with .Values.master.volumeMounts }}
{{- toYaml . | nindent 8 }}
{{- end }}
@@ -149,6 +152,8 @@ spec:
path: log4j2.xml
- key: metrics.properties
path: metrics.properties
+ - key: ratis-log4j.properties
+ path: ratis-log4j.properties
{{- with .Values.master.volumes }}
{{- toYaml . | nindent 6 }}
{{- end }}
diff --git a/charts/celeborn/tests/configmap_test.yaml
b/charts/celeborn/tests/configmap_test.yaml
index 28b793a16..fc9b09bc4 100644
--- a/charts/celeborn/tests/configmap_test.yaml
+++ b/charts/celeborn/tests/configmap_test.yaml
@@ -38,6 +38,10 @@ tests:
path: data["celeborn-defaults.conf"]
- exists:
path: data["log4j2.xml"]
+ - exists:
+ path: data["metrics.properties"]
+ - exists:
+ path: data["ratis-log4j.properties"]
- it: Should include celeborn.master.internal.endpoints key in
celeborn-defaults.conf
asserts:
diff --git a/charts/celeborn/tests/master/statefulset_test.yaml
b/charts/celeborn/tests/master/statefulset_test.yaml
index 09ef9c1d3..2601c068a 100644
--- a/charts/celeborn/tests/master/statefulset_test.yaml
+++ b/charts/celeborn/tests/master/statefulset_test.yaml
@@ -40,7 +40,7 @@ tests:
asserts:
- equal:
path:
spec.template.metadata.annotations["celeborn.apache.org/conf-hash"]
- value:
cd692d34c71437aec034c5a08728570eba1b8f5f2b20a36b153e1cd92a399481
+ value:
316cfc634ed083bd70e36ad0b429578ace6893d4de3ef4897a078db5f01a7522
- it: Should change checksum annotation when celeborn config changes
template: master/statefulset.yaml
@@ -50,10 +50,10 @@ tests:
asserts:
- notEqual:
path:
spec.template.metadata.annotations["celeborn.apache.org/conf-hash"]
- value:
cd692d34c71437aec034c5a08728570eba1b8f5f2b20a36b153e1cd92a399481
+ value:
316cfc634ed083bd70e36ad0b429578ace6893d4de3ef4897a078db5f01a7522
- equal:
path:
spec.template.metadata.annotations["celeborn.apache.org/conf-hash"]
- value:
fa6255908e6daaba8d190cb61ea2fa3f57c554279ae0dc753e56008158080d13
+ value:
fae6199c4ac5ac5a9fd9049705ac239ad6e3474a65db55bdd98f29720122b756
- it: Should add extra pod annotations if `master.annotations` is specified
template: master/statefulset.yaml
diff --git a/charts/celeborn/tests/worker/statefulset_test.yaml
b/charts/celeborn/tests/worker/statefulset_test.yaml
index d97b57009..e70dbfe3d 100644
--- a/charts/celeborn/tests/worker/statefulset_test.yaml
+++ b/charts/celeborn/tests/worker/statefulset_test.yaml
@@ -40,7 +40,7 @@ tests:
asserts:
- equal:
path:
spec.template.metadata.annotations["celeborn.apache.org/conf-hash"]
- value:
cd692d34c71437aec034c5a08728570eba1b8f5f2b20a36b153e1cd92a399481
+ value:
316cfc634ed083bd70e36ad0b429578ace6893d4de3ef4897a078db5f01a7522
- it: Should change checksum annotation when celeborn config changes
template: worker/statefulset.yaml
@@ -50,10 +50,10 @@ tests:
asserts:
- notEqual:
path:
spec.template.metadata.annotations["celeborn.apache.org/conf-hash"]
- value:
cd692d34c71437aec034c5a08728570eba1b8f5f2b20a36b153e1cd92a399481
+ value:
316cfc634ed083bd70e36ad0b429578ace6893d4de3ef4897a078db5f01a7522
- equal:
path:
spec.template.metadata.annotations["celeborn.apache.org/conf-hash"]
- value:
fa6255908e6daaba8d190cb61ea2fa3f57c554279ae0dc753e56008158080d13
+ value:
fae6199c4ac5ac5a9fd9049705ac239ad6e3474a65db55bdd98f29720122b756
- it: Should add extra pod annotations if `worker.annotations` is specified
template: worker/statefulset.yaml
