This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 100deddc34b HDDS-15289. Kubernetes check stuck cleaning up PV/PVC
(#10281)
100deddc34b is described below
commit 100deddc34bebe9d0714d1542fc5a43710598206
Author: Rishabh Patel <[email protected]>
AuthorDate: Sun May 17 00:00:46 2026 -0700
HDDS-15289. Kubernetes check stuck cleaning up PV/PVC (#10281)
---
.../definitions/ozone/definitions/persistence.yaml | 3 +++
.../examples/ozone-ha/datanode-statefulset.yaml | 3 +++
.../k8s/examples/ozone-ha/httpfs-statefulset.yaml | 3 +++
.../main/k8s/examples/ozone-ha/om-statefulset.yaml | 3 +++
.../k8s/examples/ozone-ha/recon-statefulset.yaml | 3 +++
.../k8s/examples/ozone-ha/s3g-statefulset.yaml | 3 +++
.../k8s/examples/ozone-ha/scm-statefulset.yaml | 3 +++
.../k8s/examples/ozone/datanode-statefulset.yaml | 3 +++
.../k8s/examples/ozone/httpfs-statefulset.yaml | 3 +++
.../main/k8s/examples/ozone/om-statefulset.yaml | 3 +++
.../main/k8s/examples/ozone/recon-statefulset.yaml | 3 +++
.../main/k8s/examples/ozone/s3g-statefulset.yaml | 3 +++
.../main/k8s/examples/ozone/scm-statefulset.yaml | 3 +++
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 31 ++++++++++++++++------
14 files changed, 62 insertions(+), 8 deletions(-)
diff --git
a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/persistence.yaml
b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/persistence.yaml
index 32465b68b51..8d210622ad2 100644
---
a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/persistence.yaml
+++
b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/persistence.yaml
@@ -22,6 +22,9 @@ description: Add real PVC based persistence
trigger:
kind: StatefulSet
value:
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: Delete
+ whenScaled: Retain
volumeClaimTemplates:
- metadata:
name: data
diff --git
a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/datanode-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/datanode-statefulset.yaml
index d7599c60d53..3102445ed80 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/datanode-statefulset.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/datanode-statefulset.yaml
@@ -61,6 +61,9 @@ spec:
volumeMounts:
- name: data
mountPath: /data
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: Delete
+ whenScaled: Retain
volumeClaimTemplates:
- metadata:
name: data
diff --git
a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-statefulset.yaml
index 2c076ae8fcd..967c2b1ba2c 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-statefulset.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-statefulset.yaml
@@ -50,6 +50,9 @@ spec:
volumeMounts:
- name: data
mountPath: /data
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: Delete
+ whenScaled: Retain
volumeClaimTemplates:
- metadata:
name: data
diff --git
a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/om-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/om-statefulset.yaml
index 335acfa9f92..da491f2d616 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/om-statefulset.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/om-statefulset.yaml
@@ -61,6 +61,9 @@ spec:
- name: data
mountPath: /data
volumes: []
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: Delete
+ whenScaled: Retain
volumeClaimTemplates:
- metadata:
name: data
diff --git
a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-statefulset.yaml
index 445c2e222d7..6363e7fb931 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-statefulset.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-statefulset.yaml
@@ -59,6 +59,9 @@ spec:
- name: data
mountPath: /data
volumes: []
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: Delete
+ whenScaled: Retain
volumeClaimTemplates:
- metadata:
name: data
diff --git
a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/s3g-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/s3g-statefulset.yaml
index 1cb4fa23433..1ed0deb3099 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/s3g-statefulset.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/s3g-statefulset.yaml
@@ -50,6 +50,9 @@ spec:
volumeMounts:
- name: data
mountPath: /data
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: Delete
+ whenScaled: Retain
volumeClaimTemplates:
- metadata:
name: data
diff --git
a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/scm-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/scm-statefulset.yaml
index a0be3d58edb..a1dc034d41f 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/scm-statefulset.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/scm-statefulset.yaml
@@ -80,6 +80,9 @@ spec:
volumeMounts:
- name: data
mountPath: /data
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: Delete
+ whenScaled: Retain
volumeClaimTemplates:
- metadata:
name: data
diff --git
a/hadoop-ozone/dist/src/main/k8s/examples/ozone/datanode-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/ozone/datanode-statefulset.yaml
index d7599c60d53..3102445ed80 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/datanode-statefulset.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/datanode-statefulset.yaml
@@ -61,6 +61,9 @@ spec:
volumeMounts:
- name: data
mountPath: /data
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: Delete
+ whenScaled: Retain
volumeClaimTemplates:
- metadata:
name: data
diff --git
a/hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-statefulset.yaml
index 2c076ae8fcd..967c2b1ba2c 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-statefulset.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-statefulset.yaml
@@ -50,6 +50,9 @@ spec:
volumeMounts:
- name: data
mountPath: /data
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: Delete
+ whenScaled: Retain
volumeClaimTemplates:
- metadata:
name: data
diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/om-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/ozone/om-statefulset.yaml
index 335acfa9f92..da491f2d616 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/om-statefulset.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/om-statefulset.yaml
@@ -61,6 +61,9 @@ spec:
- name: data
mountPath: /data
volumes: []
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: Delete
+ whenScaled: Retain
volumeClaimTemplates:
- metadata:
name: data
diff --git
a/hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-statefulset.yaml
index 445c2e222d7..6363e7fb931 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-statefulset.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-statefulset.yaml
@@ -59,6 +59,9 @@ spec:
- name: data
mountPath: /data
volumes: []
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: Delete
+ whenScaled: Retain
volumeClaimTemplates:
- metadata:
name: data
diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/s3g-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/ozone/s3g-statefulset.yaml
index 1cb4fa23433..1ed0deb3099 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/s3g-statefulset.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/s3g-statefulset.yaml
@@ -50,6 +50,9 @@ spec:
volumeMounts:
- name: data
mountPath: /data
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: Delete
+ whenScaled: Retain
volumeClaimTemplates:
- metadata:
name: data
diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/scm-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/ozone/scm-statefulset.yaml
index 0cbca685248..7505a017d8d 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/scm-statefulset.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/scm-statefulset.yaml
@@ -68,6 +68,9 @@ spec:
volumeMounts:
- name: data
mountPath: /data
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: Delete
+ whenScaled: Retain
volumeClaimTemplates:
- metadata:
name: data
diff --git a/hadoop-ozone/dist/src/main/k8s/examples/testlib.sh
b/hadoop-ozone/dist/src/main/k8s/examples/testlib.sh
index 81fc26f70f5..685300c22c9 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/testlib.sh
+++ b/hadoop-ozone/dist/src/main/k8s/examples/testlib.sh
@@ -90,17 +90,32 @@ pre_run_setup() {
wait_for_startup
}
+dump_pv_pvc_state() {
+ local -r label="${1:-state}"
+ echo "===== PV/PVC ${label} ====="
+ kubectl get pv,pvc -o wide --all-namespaces 2>&1 || true
+ echo "----- PV finalizers / status -----"
+ kubectl get pv -o
custom-columns=NAME:.metadata.name,STATUS:.status.phase,CLAIM:.spec.claimRef.name,FINALIZERS:.metadata.finalizers,DELETION:.metadata.deletionTimestamp
2>&1 || true
+ echo "----- Recent events (last 30) -----"
+ kubectl get events --sort-by=.lastTimestamp --all-namespaces 2>&1 | tail
-30 || true
+ echo "===== end ${label} ====="
+}
+
reset_k8s_env() {
print_phase "Deleting existing k8s resources"
#reset environment
- kubectl delete statefulset --all
- kubectl delete daemonset --all
- kubectl delete deployment --all
- kubectl delete service --all
- kubectl delete configmap --all
- kubectl delete pod --all
- kubectl delete pvc --all
- kubectl delete pv --all
+ local -r DEL_TIMEOUT="${RESET_TIMEOUT:-120s}"
+ dump_pv_pvc_state "before delete"
+ kubectl delete --timeout="$DEL_TIMEOUT" --ignore-not-found statefulset --all
+ kubectl delete --timeout="$DEL_TIMEOUT" --ignore-not-found daemonset --all
+ kubectl delete --timeout="$DEL_TIMEOUT" --ignore-not-found deployment --all
+ kubectl delete --timeout="$DEL_TIMEOUT" --ignore-not-found service --all
+ kubectl delete --timeout="$DEL_TIMEOUT" --ignore-not-found configmap --all
+ kubectl delete --timeout="$DEL_TIMEOUT" --ignore-not-found pod --all
+ kubectl delete --timeout="$DEL_TIMEOUT" --ignore-not-found pvc --all
+ dump_pv_pvc_state "after pvc delete"
+ kubectl delete --timeout="$DEL_TIMEOUT" --ignore-not-found pv --all
+ dump_pv_pvc_state "after pv delete"
}
start_k8s_env() {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
