This is an automated email from the ASF dual-hosted git repository.
jinsongzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 1aa37c1a3 [AMORO-3451] Fix kubernetes optimizer pod is frequently
being killed (#3458)
1aa37c1a3 is described below
commit 1aa37c1a3f079f4eecaf40ed2b2d39cde93fdf57
Author: Marvel-Gu <[email protected]>
AuthorDate: Fri Mar 21 03:25:39 2025 +0000
[AMORO-3451] Fix kubernetes optimizer pod is frequently being killed (#3458)
* fix: Kubernetes optimizer pod is frequently being killed
* fix: formatting code
* Improvement: provide an external parameter jvm.heap.ratio
* docs: improve documentation on jvm.heap.ratio parameter
* fix: Kubernetes optimizer pod is frequently being killed
* fix: formatting code
* Improvement: provide an external parameter jvm.heap.ratio
* docs: improve documentation on jvm.heap.ratio parameter
* Improvement: Put jvm.heap.ratio into extra to make the configuration more
flexible
* Improvement: Put jvm.heap.ratio into extra to make the configuration more
flexible
* Feature: Use an additional map to maintain the parameters in extra
* Docs: improve deployment-on-kubernetes.md and managing-optimizers.md
---
.../server/manager/KubernetesOptimizerContainer.java | 20 +++++++++++++++++++-
docs/admin-guides/deployment-on-kubernetes.md | 2 ++
docs/admin-guides/managing-optimizers.md | 2 +-
3 files changed, 22 insertions(+), 2 deletions(-)
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/manager/KubernetesOptimizerContainer.java
b/amoro-ams/src/main/java/org/apache/amoro/server/manager/KubernetesOptimizerContainer.java
index 0476c0f3a..5834e431f 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/manager/KubernetesOptimizerContainer.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/manager/KubernetesOptimizerContainer.java
@@ -73,6 +73,19 @@ public class KubernetesOptimizerContainer extends
AbstractResourceContainer {
private static final String KUBERNETES_NAME_PROPERTIES = "name";
+ private static final String EXTRA_PROPERTY_PREFIX = "extra.";
+
+ private static final Map<String, String> EXTRA_PROPERTY_DEFAULTS = new
HashMap<>();
+
+ static {
+ EXTRA_PROPERTY_DEFAULTS.put("jvm.heap.ratio", "0.8");
+ }
+
+ private String getExtraProperty(Map<String, String> properties, String key) {
+ return properties.getOrDefault(
+ EXTRA_PROPERTY_PREFIX + key, EXTRA_PROPERTY_DEFAULTS.getOrDefault(key,
null));
+ }
+
private KubernetesClient client;
@Override
@@ -148,6 +161,7 @@ public class KubernetesOptimizerContainer extends
AbstractResourceContainer {
Resource resource, Map<String, String> groupProperties) {
long memoryPerThread;
long memory;
+ long jvmHeapMemory;
if (resource.getMemoryMb() > 0) {
memory = resource.getMemoryMb();
@@ -155,11 +169,15 @@ public class KubernetesOptimizerContainer extends
AbstractResourceContainer {
memoryPerThread = Long.parseLong(checkAndGetProperty(groupProperties,
MEMORY_PROPERTY));
memory = memoryPerThread * resource.getThreadCount();
}
+
+ double jvmHeapRatio = Double.parseDouble(getExtraProperty(groupProperties,
"jvm.heap.ratio"));
+ jvmHeapMemory = (long) (memory * jvmHeapRatio);
+
// point at amoro home in docker image
String startUpArgs =
String.format(
"/entrypoint.sh optimizer %s %s",
- memory, super.buildOptimizerStartupArgsString(resource));
+ jvmHeapMemory, super.buildOptimizerStartupArgsString(resource));
LOG.info("Starting k8s optimizer using k8s client with start command :
{}", startUpArgs);
String namespace = groupProperties.getOrDefault(NAMESPACE, "default");
diff --git a/docs/admin-guides/deployment-on-kubernetes.md
b/docs/admin-guides/deployment-on-kubernetes.md
index 1d7b4b00a..1211bdeb2 100644
--- a/docs/admin-guides/deployment-on-kubernetes.md
+++ b/docs/admin-guides/deployment-on-kubernetes.md
@@ -224,6 +224,8 @@ optimizer:
kube-config-path: "~/.kube/config"
image: "apache/amoro:latest"
pullPolicy: "IfNotPresent"
+ # configure additional parameters by using the extra. prefix
+ # extra.jvm.heap.ratio: "0.8"
```
To use PodTemplate, you need to copy and paste the following into the
`kubernetes.properties`.
diff --git a/docs/admin-guides/managing-optimizers.md
b/docs/admin-guides/managing-optimizers.md
index 71f4c313d..f168579c6 100644
--- a/docs/admin-guides/managing-optimizers.md
+++ b/docs/admin-guides/managing-optimizers.md
@@ -85,7 +85,7 @@ Kubernetes container support the following properties:
| ams-optimizing-uri | false | N/A | URI of AMS thrift
self-optimizing endpoint. This could be used if the ams.server-expose-host is
not available |
| cpu.factor | false | "1.0" | CPU factor when
request kubernetes resource. Default 1 Cpu pre thread
|
| memory | true | N/A | Memory usage for pre
thread
|
-
+| extra.jvm.heap.ratio | false | 0.8 | The ratio of JVM heap
memory to total pod memory
|
```yaml
containers:
