This is an automated email from the ASF dual-hosted git repository.
mchades pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino-playground.git
The following commit(s) were added to refs/heads/main by this push:
new b941c25 [#142] improvement: Use environment variables to specify the
version (#143)
b941c25 is described below
commit b941c256d2e12e2141681515914f8f1fb596b7b0
Author: mchades <[email protected]>
AuthorDate: Wed Jul 23 17:27:28 2025 +0800
[#142] improvement: Use environment variables to specify the version (#143)
### What changes were proposed in this pull request?
Use environment variables to specify the version
### Why are the changes needed?
Fix: #142
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
by hand
---
.env | 16 +++++++++++++
docker-compose.yaml | 28 ++++++++++++----------
.../gravitino-access-control-example.ipynb | 10 ++++----
init/jupyter/gravitino-spark-trino-example.ipynb | 3 ++-
init/spark/init.sh | 2 +-
init/spark/spark-dependency.sh | 2 +-
6 files changed, 42 insertions(+), 19 deletions(-)
diff --git a/.env b/.env
new file mode 100644
index 0000000..14c6aad
--- /dev/null
+++ b/.env
@@ -0,0 +1,16 @@
+# Image tags for the docker-compose file
+HIVE_IMAGE_TAG=hive-0.1.15
+RANGER_IMAGE_TAG=ranger-0.1.0
+GRAVITINO_IMAGE_TAG=0.9.0-incubating
+TRINO_IMAGE_TAG=trino-435-gravitino-0.9.0-incubating
+POSTGRESQL_IMAGE_TAG=13
+MYSQL_IMAGE_TAG=8.0
+SPARK_IMAGE_TAG=3.4.1-scala2.12-java11-ubuntu
+JUPYTER_IMAGE_TAG=spark-3.4.1
+PROMETHEUS_IMAGE_TAG=v2.55.0
+GRAFANA_IMAGE_TAG=11.3.0-ubuntu
+
+# Gravitino jars for containers
+SPARK_CONNECTOR_JAR=gravitino-spark-connector-runtime-3.4_2.12-0.9.0-incubating.jar
+
+GRAVITINO_VERSION=0.9.0-incubating
\ No newline at end of file
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 79f0c22..b57a6cc 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -18,7 +18,7 @@
#
services:
hive:
- image: apache/gravitino-playground:hive-0.1.15
+ image: apache/gravitino-playground:${HIVE_IMAGE_TAG}
ports:
- "3307:3306"
- "19000:9000"
@@ -38,7 +38,7 @@ services:
start_period: 20s
ranger:
- image: apache/gravitino-playground:ranger-0.1.0
+ image: apache/gravitino-playground:${RANGER_IMAGE_TAG}
ports:
- "6080:6080"
container_name: playground-ranger
@@ -54,16 +54,16 @@ services:
start_period: 120s
gravitino:
- image: apache/gravitino:0.9.0-incubating
+ image: apache/gravitino:${GRAVITINO_IMAGE_TAG}
entrypoint: /bin/bash /tmp/gravitino/init.sh
ports:
- "8090:8090"
- "9001:9001"
container_name: playground-gravitino
depends_on:
- ranger :
+ ranger:
condition: service_healthy
- hive :
+ hive:
condition: service_healthy
mysql:
condition: service_healthy
@@ -79,7 +79,7 @@ services:
start_period: 20s
trino:
- image: apache/gravitino-playground:trino-435-gravitino-0.9.0-incubating
+ image: apache/gravitino-playground:${TRINO_IMAGE_TAG}
ports:
- "18080:8080"
container_name: playground-trino
@@ -109,7 +109,7 @@ services:
start_period: 20s
postgresql:
- image: postgres:13
+ image: postgres:${POSTGRESQL_IMAGE_TAG}
container_name: playground-postgresql
restart: always
environment:
@@ -122,7 +122,7 @@ services:
- ./init/postgres:/docker-entrypoint-initdb.d/
mysql:
- image: mysql:8.0
+ image: mysql:${MYSQL_IMAGE_TAG}
container_name: playground-mysql
environment:
- MYSQL_ROOT_PASSWORD=mysql
@@ -148,7 +148,7 @@ services:
start_period: 20s
spark:
- image: spark:3.4.1-scala2.12-java11-ubuntu
+ image: spark:${SPARK_IMAGE_TAG}
container_name: playground-spark
depends_on:
gravitino:
@@ -158,6 +158,8 @@ services:
- HADOOP_USER_NAME=root
- HIVE_HOST_IP=hive
- MYSQL_HOST_IP=mysql
+ - SPARK_CONNECTOR_JAR=${SPARK_CONNECTOR_JAR}
+ - GRAVITINO_VERSION=${GRAVITINO_VERSION}
ports:
- "14040:4040"
volumes:
@@ -165,7 +167,7 @@ services:
- ./init/common:/tmp/common
jupyter:
- image: jupyter/pyspark-notebook:spark-3.4.1
+ image: jupyter/pyspark-notebook:${JUPYTER_IMAGE_TAG}
container_name: playground-jupyter
ports:
- "18888:8888"
@@ -173,6 +175,8 @@ services:
- ./init/jupyter:/tmp/gravitino
- ./data/jupyter/data:/home/jovyan
entrypoint: /bin/bash /tmp/gravitino/init.sh
+ environment:
+ - SPARK_CONNECTOR_JAR=${SPARK_CONNECTOR_JAR}
depends_on:
hive:
condition: service_healthy
@@ -180,7 +184,7 @@ services:
condition: service_healthy
prometheus:
- image: prom/prometheus:v2.55.0
+ image: prom/prometheus:${PROMETHEUS_IMAGE_TAG}
container_name: playground-prometheus
ports:
- "19090:9090"
@@ -188,7 +192,7 @@ services:
- ./init/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
grafana:
- image: grafana/grafana:11.3.0-ubuntu
+ image: grafana/grafana:${GRAFANA_IMAGE_TAG}
container_name: playground-grafana
ports:
- "13000:3000"
diff --git a/init/jupyter/authorization/gravitino-access-control-example.ipynb
b/init/jupyter/authorization/gravitino-access-control-example.ipynb
index b06141f..4cda534 100644
--- a/init/jupyter/authorization/gravitino-access-control-example.ipynb
+++ b/init/jupyter/authorization/gravitino-access-control-example.ipynb
@@ -129,12 +129,13 @@
"import os\n",
"from pyspark.sql import SparkSession\n",
"os.environ['HADOOP_USER_NAME']=\"manager\"\n",
+ "gravitino_connector_jar = os.getenv('SPARK_CONNECTOR_JAR')\n",
"\n",
"spark = SparkSession.builder \\\n",
" .appName(\"PySpark SQL Example\") \\\n",
" .config(\"spark.plugins\",
\"org.apache.gravitino.spark.connector.plugin.GravitinoSparkPlugin\") \\\n",
- " .config(\"spark.jars\",
\"/tmp/gravitino/packages/iceberg-spark-runtime-3.4_2.12-1.5.2.jar,\\\n",
- "
/tmp/gravitino/packages/gravitino-spark-connector-runtime-3.4_2.12-0.9.0-incubating.jar,\\\n",
+ " .config(\"spark.jars\",
f\"/tmp/gravitino/packages/iceberg-spark-runtime-3.4_2.12-1.5.2.jar,\\\n",
+ "
/tmp/gravitino/packages/{gravitino_connector_jar},\\\n",
"
/tmp/gravitino/packages/kyuubi-spark-authz-shaded_2.12-1.9.2.jar\") \\\n",
" .config(\"spark.sql.gravitino.uri\", \"http://gravitino:8090\";) \\\n",
" .config(\"spark.sql.gravitino.metalake\", \"metalake_demo\") \\\n",
@@ -258,12 +259,13 @@
"import os\n",
"from pyspark.sql import SparkSession\n",
"os.environ['HADOOP_USER_NAME']=\"lisa\"\n",
+ "gravitino_connector_jar = os.getenv('SPARK_CONNECTOR_JAR')\n",
"\n",
"spark = SparkSession.builder \\\n",
" .appName(\"PySpark SQL Example\") \\\n",
" .config(\"spark.plugins\",
\"org.apache.gravitino.spark.connector.plugin.GravitinoSparkPlugin\") \\\n",
- " .config(\"spark.jars\",
\"/tmp/gravitino/packages/iceberg-spark-runtime-3.4_2.12-1.5.2.jar,\\\n",
- "
/tmp/gravitino/packages/gravitino-spark-connector-runtime-3.4_2.12-0.9.0-incubating.jar,\\\n",
+ " .config(\"spark.jars\",
f\"/tmp/gravitino/packages/iceberg-spark-runtime-3.4_2.12-1.5.2.jar,\\\n",
+ "
/tmp/gravitino/packages/{gravitino_connector_jar},\\\n",
"
/tmp/gravitino/packages/kyuubi-spark-authz-shaded_2.12-1.9.2.jar\") \\\n",
" .config(\"spark.sql.gravitino.uri\", \"http://gravitino:8090\";) \\\n",
" .config(\"spark.sql.gravitino.metalake\", \"metalake_demo\") \\\n",
diff --git a/init/jupyter/gravitino-spark-trino-example.ipynb
b/init/jupyter/gravitino-spark-trino-example.ipynb
index 011c325..c0e0db1 100644
--- a/init/jupyter/gravitino-spark-trino-example.ipynb
+++ b/init/jupyter/gravitino-spark-trino-example.ipynb
@@ -20,12 +20,13 @@
"from pyspark.sql import SparkSession\n",
"\n",
"spark_home = os.getenv('SPARK_HOME')\n",
+ "gravitino_connector_jar = os.getenv('SPARK_CONNECTOR_JAR')\n",
"os.environ['HADOOP_USER_NAME']=\"anonymous\"\n",
"\n",
"spark = SparkSession.builder \\\n",
" .appName(\"PySpark SQL Example\") \\\n",
" .config(\"spark.plugins\",
\"org.apache.gravitino.spark.connector.plugin.GravitinoSparkPlugin\") \\\n",
- " .config(\"spark.jars\",
\"/tmp/gravitino/packages/iceberg-spark-runtime-3.4_2.12-1.5.2.jar,/tmp/gravitino/packages/gravitino-spark-connector-runtime-3.4_2.12-0.9.0-incubating.jar\")
\\\n",
+ " .config(\"spark.jars\",
f\"/tmp/gravitino/packages/iceberg-spark-runtime-3.4_2.12-1.5.2.jar,/tmp/gravitino/packages/{gravitino_connector_jar}\")
\\\n",
" .config(\"spark.sql.gravitino.uri\", \"http://gravitino:8090\";) \\\n",
" .config(\"spark.sql.gravitino.metalake\", \"metalake_demo\") \\\n",
" .config(\"spark.sql.gravitino.enableIcebergSupport\", \"true\") \\\n",
diff --git a/init/spark/init.sh b/init/spark/init.sh
index 26ec63f..3d62689 100644
--- a/init/spark/init.sh
+++ b/init/spark/init.sh
@@ -21,7 +21,7 @@ cp /tmp/spark/spark-defaults.conf /opt/spark/conf
cp /tmp/spark/spark-env.sh /opt/spark/conf
cp /tmp/spark/packages/iceberg-spark-runtime-3.4_2.12-1.5.2.jar
/opt/spark/jars/iceberg-spark-runtime-3.4_2.12-1.5.2.jar
-cp
/tmp/spark/packages/gravitino-spark-connector-runtime-3.4_2.12-0.9.0-incubating.jar
/opt/spark/jars/gravitino-spark-connector-runtime-3.4_2.12-0.9.0-incubating.jar
+cp /tmp/spark/packages/${SPARK_CONNECTOR_JAR}
/opt/spark/jars/${SPARK_CONNECTOR_JAR}
cp /tmp/spark/packages/mysql-connector-java-8.0.27.jar
/opt/spark/jars/mysql-connector-java-8.0.27.jar
cp /tmp/spark/packages/kyuubi-spark-authz-shaded_2.12-1.9.2.jar
/opt/spark/jars/kyuubi-spark-authz-shaded_2.12-1.9.2.jar
sh /tmp/common/init_metalake_catalog.sh
diff --git a/init/spark/spark-dependency.sh b/init/spark/spark-dependency.sh
index 0af2fae..0cfb844 100755
--- a/init/spark/spark-dependency.sh
+++ b/init/spark/spark-dependency.sh
@@ -33,7 +33,7 @@
ICEBERG_SPARK_RUNTIME_JAR="https://repo1.maven.org/maven2/org/apache/iceberg/ice
ICEBERG_SPARK_RUNTIME_MD5="${ICEBERG_SPARK_RUNTIME_JAR}.md5"
download_and_verify "${ICEBERG_SPARK_RUNTIME_JAR}"
"${ICEBERG_SPARK_RUNTIME_MD5}" "${spark_dir}"
-GRAVITINO_SPARK_CONNECTOR_RUNTIME_JAR="https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-spark-connector-runtime-3.4_2.12/0.9.0-incubating/gravitino-spark-connector-runtime-3.4_2.12-0.9.0-incubating.jar";
+GRAVITINO_SPARK_CONNECTOR_RUNTIME_JAR="https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-spark-connector-runtime-3.4_2.12/${GRAVITINO_VERSION}/${GRAVITINO_SPARK_CONNECTOR_RUNTIME_JAR}";
GRAVITINO_SPARK_CONNECTOR_RUNTIME_MD5="${GRAVITINO_SPARK_CONNECTOR_RUNTIME_JAR}.md5"
download_and_verify "${GRAVITINO_SPARK_CONNECTOR_RUNTIME_JAR}"
"${GRAVITINO_SPARK_CONNECTOR_RUNTIME_MD5}" "${spark_dir}"
