This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 6de1ae69cf [#9042] Add postgresql backend in chart (#9051)
6de1ae69cf is described below
commit 6de1ae69cf535db81129eb70e7f4bb523c4c53c3
Author: Danhua Wang <[email protected]>
AuthorDate: Tue Nov 18 01:24:39 2025 +0800
[#9042] Add postgresql backend in chart (#9051)
### What changes were proposed in this pull request?
1. Add Postgresql backend deployment in gravitino chart
2. Add Postgresql backend CI in chart testing workflow
3. Lack Postgresql driver in Gravitino libs directory, add it in
Dockerfile
4. Remove volumeClaimTemplates section due to it's not suitable for k8s
Deployment resources. Additionally it's used for k8s Statefulset
resources.
### Why are the changes needed?
Fix: #9042
### Does this PR introduce _any_ user-facing change?
Gravitino chart can deploy Postgresql instance and use it as Gravitino
storage backend.
### How was this patch tested?
1. Chart-testing CI
2. Manually test: deploying Gravitino chart (0.9.1) with Postgresql
instance , then upgrading gravitino from 0.9.1 to 1.1.0 by `helm
upgrade` command. Check `upgrade-0.9.0-to-1.0.0-mysql.sql` working
correctly in `init-postgresql` container.
---
.github/workflows/chart-test.yaml | 6 +-
dev/charts/gravitino/Chart.yaml | 4 ++
.../gravitino/resources/config/gravitino.conf | 8 +--
.../gravitino/resources/scenarios/pg-values.yaml | 26 ++++++++
dev/charts/gravitino/templates/_helpers.tpl | 9 +++
dev/charts/gravitino/templates/deployment.yaml | 72 +++++++++++++++++-----
dev/charts/gravitino/values.yaml | 38 ++++++++++++
dev/docker/gravitino/gravitino-dependency.sh | 1 +
8 files changed, 142 insertions(+), 22 deletions(-)
diff --git a/.github/workflows/chart-test.yaml
b/.github/workflows/chart-test.yaml
index 1971e405b7..2dfd9a946f 100644
--- a/.github/workflows/chart-test.yaml
+++ b/.github/workflows/chart-test.yaml
@@ -82,4 +82,8 @@ jobs:
- name: Run chart-testing (install mysql enable)
if: steps.list-changed.outputs.changed == 'true'
- run: ct install --charts=dev/charts/gravitino --helm-extra-set-args
"--values dev/charts/gravitino/resources/scenarios/ci-values.yaml"
--target-branch ${{ github.event.pull_request.base.ref }}
\ No newline at end of file
+ run: ct install --charts=dev/charts/gravitino --helm-extra-set-args
"--values dev/charts/gravitino/resources/scenarios/ci-values.yaml"
--target-branch ${{ github.event.pull_request.base.ref }}
+
+ - name: Run chart-testing (install postgresql enable)
+ if: steps.list-changed.outputs.changed == 'true'
+ run: ct install --charts=dev/charts/gravitino --helm-extra-set-args
"--values dev/charts/gravitino/resources/scenarios/pg-values.yaml"
--target-branch ${{ github.event.pull_request.base.ref }}
\ No newline at end of file
diff --git a/dev/charts/gravitino/Chart.yaml b/dev/charts/gravitino/Chart.yaml
index 02e4c90eb9..8a65177646 100644
--- a/dev/charts/gravitino/Chart.yaml
+++ b/dev/charts/gravitino/Chart.yaml
@@ -32,6 +32,10 @@ dependencies:
name: mysql
repository: https://charts.bitnami.com/bitnami
version: 10.2.1
+ - condition: postgresql.enabled
+ name: postgresql
+ repository: https://charts.bitnami.com/bitnami
+ version: 15.4.2
kubeVersion: '>=1.28.0-0'
maintainers:
- name: Gravitino
diff --git a/dev/charts/gravitino/resources/config/gravitino.conf
b/dev/charts/gravitino/resources/config/gravitino.conf
index f237afe321..0d714b9ff8 100644
--- a/dev/charts/gravitino/resources/config/gravitino.conf
+++ b/dev/charts/gravitino/resources/config/gravitino.conf
@@ -43,10 +43,10 @@ gravitino.entity.store.maxTransactionSkewTimeMs = {{
.Values.entity.maxTransacti
gravitino.entity.store.deleteAfterTimeMs = {{
(.Values.entity.deleteAfterTimeMs | default 604800000) | int }}
gravitino.entity.store.versionRetentionCount = {{
.Values.entity.versionRetentionCount }}
gravitino.entity.store.relational = {{ .Values.entity.relationalBackend |
default "JDBCBackend" }}
-gravitino.entity.store.relational.jdbcUrl = {{ if .Values.mysql.enabled
}}jdbc:mysql://{{ .Release.Name }}-mysql:3306/{{ .Values.mysql.auth.database
}}{{ else }}{{ .Values.entity.jdbcUrl }}{{ end }}
-gravitino.entity.store.relational.jdbcDriver = {{ if .Values.mysql.enabled
}}com.mysql.cj.jdbc.Driver{{ else }}{{ .Values.entity.jdbcDriver }}{{ end }}
-gravitino.entity.store.relational.jdbcUser = {{ if .Values.mysql.enabled }}{{
.Values.mysql.auth.username }}{{ else }}{{ .Values.entity.jdbcUser }}{{ end }}
-gravitino.entity.store.relational.jdbcPassword = {{ if .Values.mysql.enabled
}}{{ .Values.mysql.auth.password }}{{ else }}{{ .Values.entity.jdbcPassword
}}{{ end }}
+gravitino.entity.store.relational.jdbcUrl = {{ if .Values.mysql.enabled
}}jdbc:mysql://{{ .Release.Name }}-mysql:3306/{{ .Values.mysql.auth.database
}}{{ else if .Values.postgresql.enabled }}jdbc:postgresql://{{ .Release.Name
}}-postgresql:5432/{{ .Values.postgresql.auth.database }}{{ else }}{{
.Values.entity.jdbcUrl }}{{ end }}
+gravitino.entity.store.relational.jdbcDriver = {{ if .Values.mysql.enabled
}}com.mysql.cj.jdbc.Driver{{ else if .Values.postgresql.enabled
}}org.postgresql.Driver{{ else }}{{ .Values.entity.jdbcDriver }}{{ end }}
+gravitino.entity.store.relational.jdbcUser = {{ if .Values.mysql.enabled }}{{
.Values.mysql.auth.username }}{{ else if .Values.postgresql.enabled }}{{
.Values.postgresql.auth.username }}{{ else }}{{ .Values.entity.jdbcUser }}{{
end }}
+gravitino.entity.store.relational.jdbcPassword = {{ if .Values.mysql.enabled
}}{{ .Values.mysql.auth.password }}{{ else if .Values.postgresql.enabled }}{{
.Values.postgresql.auth.password }}{{ else }}{{ .Values.entity.jdbcPassword
}}{{ end }}
gravitino.entity.store.relational.storagePath = {{ .Values.entity.storagePath
}}
# THE CONFIGURATION FOR Gravitino CATALOG
diff --git a/dev/charts/gravitino/resources/scenarios/pg-values.yaml
b/dev/charts/gravitino/resources/scenarios/pg-values.yaml
new file mode 100644
index 0000000000..0321f78c6b
--- /dev/null
+++ b/dev/charts/gravitino/resources/scenarios/pg-values.yaml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+postgresql:
+ enabled: true
+
+initScript: |
+ cp /tmp/conf/* ${GRAVITINO_HOME}/conf
+ wget https://jdbc.postgresql.org/download/postgresql-42.7.0.jar -O
/root/gravitino/libs/postgresql-42.7.0.jar
+ echo "Start the Gravitino Server"
+ /bin/bash ${GRAVITINO_HOME}/bin/gravitino.sh run
\ No newline at end of file
diff --git a/dev/charts/gravitino/templates/_helpers.tpl
b/dev/charts/gravitino/templates/_helpers.tpl
index 8258ac350d..9dc4f0b560 100644
--- a/dev/charts/gravitino/templates/_helpers.tpl
+++ b/dev/charts/gravitino/templates/_helpers.tpl
@@ -80,4 +80,13 @@ Return the proper MySQL image name for the init-mysql
initContainer
{{- if .Values.mysql.enabled -}}
{{- include "common.images.image" (dict "imageRoot" .Values.mysql.image
"global" .Values.global) -}}
{{- end -}}
+{{- end -}}
+
+{{/*
+Return the proper Postgresql image name for the init-postgresql initContainer
+*/}}
+{{- define "init-postgresql.image" -}}
+{{- if .Values.postgresql.enabled -}}
+ {{- include "common.images.image" (dict "imageRoot" .Values.postgresql.image
"global" .Values.global) -}}
+{{- end -}}
{{- end -}}
\ No newline at end of file
diff --git a/dev/charts/gravitino/templates/deployment.yaml
b/dev/charts/gravitino/templates/deployment.yaml
index 700df9463b..ddf498c52d 100644
--- a/dev/charts/gravitino/templates/deployment.yaml
+++ b/dev/charts/gravitino/templates/deployment.yaml
@@ -97,11 +97,11 @@ spec:
fi
echo "Schema file: $schema_file"
echo "Upgrade file: $upgrade_file"
- if ! mysql -h {{ .Release.Name }}-mysql -u root -D gravitino
<"$upgrade_file"; then
+ if ! mysql -h {{ .Release.Name }}-mysql -u root -D $GRAVITINO_DB
<"$upgrade_file"; then
echo "WARNING: Upgrade script failed: $upgrade_file" >&2
echo "Continuing with schema script..." >&2
fi
- mysql -h {{ .Release.Name }}-mysql -u root -D gravitino
<$schema_file
+ mysql -h {{ .Release.Name }}-mysql -u root -D $GRAVITINO_DB
<$schema_file
resources:
{{- toYaml .Values.initResources | nindent 10 }}
env:
@@ -110,10 +110,63 @@ spec:
secretKeyRef:
name: {{ .Release.Name }}-mysql
key: mysql-root-password
+ - name: GRAVITINO_DB
+ value: {{ .Values.mysql.auth.database }}
volumeMounts:
- mountPath: /scripts
name: scripts-emptydir
{{- end }}
+ {{- if .Values.postgresql.enabled }}
+ - name: init-postgresql
+ image: {{ include "init-postgresql.image" . }}
+ imagePullPolicy: "{{ .Values.image.pullPolicy }}"
+ command:
+ - /bin/bash
+ - -c
+ - |
+ while ! PGPASSWORD=$POSTGRES_PASSWORD psql -h {{ .Release.Name
}}-postgresql -U $POSTGRES_USER -c "SELECT 1"; do
+ echo "Waiting for PostgreSQL to be ready..."
+ sleep 2
+ done
+ VERSION=$(cat /scripts/version.txt)
+ if [ -z "$VERSION" ]; then
+ echo "ERROR:Please set VERSION"
+ exit 1
+ fi
+ dir="/scripts/postgresql"
+ schema_file=$(ls $dir/schema-*-postgresql.sql | sort -V | awk
-F'[-]' '$2 <= '$VERSION' {print $0}' | tail -n 1)
+ if [ ! -f "$schema_file" ]; then
+ echo "ERROR:Failed to find schema file $schema_file"
+ exit 1
+ fi
+ upgrade_file=$(ls $dir/upgrade-*-to-*-postgresql.sql | sort -V |
awk -F'[-]' '$4 <= '$VERSION' {print $0}' | tail -n 1)
+ if [ ! -f "$upgrade_file" ]; then
+ echo "ERROR:Failed to find upgrade file $upgrade_file"
+ exit 1
+ fi
+ echo "Schema file: $schema_file"
+ echo "Upgrade file: $upgrade_file"
+ if ! PGPASSWORD=$POSTGRES_PASSWORD psql -h {{ .Release.Name
}}-postgresql -U $POSTGRES_USER -d $GRAVITINO_DB -f "$upgrade_file"; then
+ echo "WARNING: Upgrade script failed: $upgrade_file" >&2
+ echo "Continuing with schema script..." >&2
+ fi
+ PGPASSWORD=$POSTGRES_PASSWORD psql -h {{ .Release.Name
}}-postgresql -U $POSTGRES_USER -d $GRAVITINO_DB -f "$schema_file"
+ resources:
+ {{- toYaml .Values.initResources | nindent 10 }}
+ env:
+ - name: POSTGRES_USER
+ value: {{ .Values.postgresql.auth.username }}
+ - name: GRAVITINO_DB
+ value: {{ .Values.postgresql.auth.database }}
+ - name: POSTGRES_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Release.Name }}-postgresql
+ key: password
+ volumeMounts:
+ - mountPath: /scripts
+ name: scripts-emptydir
+ {{- end }}
containers:
- name: {{ include "gravitino.name" . }}
image: {{ include "gravitino.image" . }}
@@ -181,21 +234,6 @@ spec:
- name: storage
persistentVolumeClaim:
claimName: {{ .Values.persistence.existingClaim }}
- {{- else }}
- volumeClaimTemplates:
- - metadata:
- name: storage
- labels:
- {{- toYaml .Values.persistence.labels | nindent 8 }}
- annotations:
- {{- toYaml .Values.persistence.annotations | nindent 8 }}
- spec:
- accessModes:
- {{- toYaml .Values.persistence.accessModes | nindent 8 }}
- resources:
- requests:
- storage: {{ .Values.persistence.size | quote }}
- storageClassName: {{ .Values.persistence.storageClassName }}
{{- end }}
{{- if .Values.extraVolumes }}
{{- toYaml .Values.extraVolumes | nindent 8 }}
diff --git a/dev/charts/gravitino/values.yaml b/dev/charts/gravitino/values.yaml
index cfa7df7904..6c96ca99c9 100644
--- a/dev/charts/gravitino/values.yaml
+++ b/dev/charts/gravitino/values.yaml
@@ -69,6 +69,44 @@ mysql:
##
password: gravitino
+## PostgreSQL chart configuration
+## ref:
https://github.com/bitnami/charts/blob/master/bitnami/postgresql/values.yaml
+##
+postgresql:
+ ## @param postgresql.enabled Deploy PostgreSQL container(s)
+ ##
+ enabled: false
+ ## Bitnami PostgreSQL image version
+ ## ref: https://hub.docker.com/r/bitnami/postgresql/tags/
+ ## @param image.registry PostgreSQL image registry
+ ## @param image.repository PostgreSQL image repository
+ ## @param image.tag PostgreSQL image tag (immutable tags are recommended)
+ ##
+ image:
+ registry: docker.io
+ repository: bitnamilegacy/postgresql
+ tag: 16.3.0-debian-12-r10
+ auth:
+ ## @param auth.username Name for a custom user to create
+ ##
+ username: gravitino
+ ## @param auth.password Password for the custom user to create
+ ##
+ password: gravitino
+ ## @param auth.database Name for a custom database to create
+ ##
+ database: gravitino
+ ## @param auth.existingSecret Name of existing secret to use for
PostgreSQL credentials
+ ## `auth.postgresPassword`, `auth.password`, and
`auth.replicationPassword` will be ignored and picked up from this secret
+ ## The secret must contain the keys `postgres-password` (which is the
password for "postgres" admin user),
+ ## `password` (which is the password for the custom user to create when
`auth.username` is set),
+ ## and `replication-password` (which is the password for replication user).
+ ## The secret might also contains the key `ldap-password` if LDAP is
enabled. `ldap.bind_password` will be ignored and
+ ## picked from this secret in this case.
+ ## The value is evaluated as a template.
+ ##
+ existingSecret: ""
+
## THE CONFIGURATION FOR Gravitino ENTITY STORE
##
entity:
diff --git a/dev/docker/gravitino/gravitino-dependency.sh
b/dev/docker/gravitino/gravitino-dependency.sh
index aab5bcec7c..32429950f6 100755
--- a/dev/docker/gravitino/gravitino-dependency.sh
+++ b/dev/docker/gravitino/gravitino-dependency.sh
@@ -94,6 +94,7 @@ wget "https://jdbc.postgresql.org/download/${pg_driver}"; -O
"${gravitino_staging
cp "${gravitino_staging_dir}/${pg_driver}"
"${gravitino_package_dir}/catalogs/jdbc-postgresql/libs/"
cp "${gravitino_staging_dir}/${pg_driver}"
"${gravitino_package_dir}/catalogs/lakehouse-iceberg/libs/"
cp "${gravitino_staging_dir}/${pg_driver}" "${gravitino_iceberg_rest_dir}"
+cp "${gravitino_staging_dir}/${pg_driver}" "${gravitino_package_dir}/libs/"
iceberg_version="1.10.0"
iceberg_aws_bundle="iceberg-aws-bundle-${iceberg_version}.jar"
