This is an automated email from the ASF dual-hosted git repository.
ChenSammi pushed a commit to branch HDDS-13323-sts
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-13323-sts by this push:
new 3a56b7faa35 HDDS-15325. [STS] Polaris Smoke Test (#10315)
3a56b7faa35 is described below
commit 3a56b7faa3591d36b5922ff2eff2b4436670eee6
Author: fmorg-git <[email protected]>
AuthorDate: Mon Aug 17 22:30:47 2026 -0700
HDDS-15325. [STS] Polaris Smoke Test (#10315)
---
.../main/compose/ozonesecure-ha/polaris-setup.sh | 123 +++++++++++++++++++++
.../compose/ozonesecure-ha/polaris-smoketest.sh | 113 +++++++++++++++++++
.../src/main/compose/ozonesecure-ha/polaris.yaml | 77 +++++++++++++
.../src/main/compose/ozonesecure-ha/test-ranger.sh | 1 +
.../security/ozone-secure-sts-polaris.sql | 27 +++++
5 files changed, 341 insertions(+)
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/polaris-setup.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/polaris-setup.sh
new file mode 100755
index 00000000000..aab766f270e
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/polaris-setup.sh
@@ -0,0 +1,123 @@
+#!/usr/bin/env sh
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+apk add --no-cache jq >/dev/null
+
+realm="${POLARIS_REALM:-POLARIS}"
+catalog_name="${POLARIS_CATALOG_NAME:-quickstart_catalog}"
+storage_location="${POLARIS_STORAGE_LOCATION:-s3://iceberg-obs/polaris-smoke}"
+s3_endpoint="${POLARIS_S3_ENDPOINT:-http://s3g:9878}";
+sts_endpoint="${POLARIS_STS_ENDPOINT:-http://s3g:9880/sts}";
+role_arn="${POLARIS_ROLE_ARN:-arn:aws:iam::123456789012:role/iceberg-data-all-access-obs}"
+
+if [ -z "${POLARIS_AWS_ACCESS_KEY_ID:-}" ] || [ -z
"${POLARIS_AWS_SECRET_ACCESS_KEY:-}" ]; then
+ echo "POLARIS_AWS_ACCESS_KEY_ID and POLARIS_AWS_SECRET_ACCESS_KEY must be
set"
+ exit 1
+fi
+
+echo "Waiting for S3 gateway at ${s3_endpoint}..."
+attempt=0
+while [ "${attempt}" -lt 30 ]; do
+ if curl --silent --show-error --include \
+ --user "${POLARIS_AWS_ACCESS_KEY_ID}:${POLARIS_AWS_SECRET_ACCESS_KEY}" \
+ --aws-sigv4 "aws:amz:us-west-2:s3" \
+ "${s3_endpoint}/" >/dev/null 2>&1; then
+ echo "${s3_endpoint} is available"
+ break
+ fi
+ attempt=$((attempt + 1))
+ sleep 2
+done
+if [ "${attempt}" -ge 30 ]; then
+ echo "Timed out waiting for S3 gateway at ${s3_endpoint}"
+ exit 1
+fi
+
+echo "Obtaining Polaris OAuth token..."
+token="$(
+ curl --fail-with-body --silent \
+ --user "${CLIENT_ID}:${CLIENT_SECRET}" \
+ -H "Polaris-Realm: ${realm}" \
+ -d grant_type=client_credentials \
+ -d scope=PRINCIPAL_ROLE:ALL \
+ "http://polaris:8181/api/catalog/v1/oauth/tokens"; \
+ | jq -r .access_token
+)"
+if [ -z "${token}" ] || [ "${token}" = "null" ]; then
+ echo "Failed to obtain access token."
+ exit 1
+fi
+
+storage_config_info="$(
+ jq -n \
+ --arg endpoint "${s3_endpoint}" \
+ --arg endpointInternal "${s3_endpoint}" \
+ --arg stsEndpoint "${sts_endpoint}" \
+ --arg roleArn "${role_arn}" \
+ '{
+ storageType: "S3",
+ endpoint: $endpoint,
+ endpointInternal: $endpointInternal,
+ stsEndpoint: $stsEndpoint,
+ roleArn: $roleArn,
+ stsUnavailable: false,
+ pathStyleAccess: true,
+ region: "us-west-2"
+ }'
+)"
+
+payload="$(
+ jq -n \
+ --arg name "${catalog_name}" \
+ --arg location "${storage_location}" \
+ --argjson storageConfigInfo "${storage_config_info}" \
+ '{
+ catalog: {
+ name: $name,
+ type: "INTERNAL",
+ readOnly: false,
+ properties: {
+ "default-base-location": $location
+ },
+ storageConfigInfo: $storageConfigInfo
+ }
+ }'
+)"
+
+echo "Creating catalog ${catalog_name} in realm ${realm}..."
+curl --fail-with-body --silent \
+ -H "Authorization: Bearer ${token}" \
+ -H "Accept: application/json" \
+ -H "Content-Type: application/json" \
+ -H "Polaris-Realm: ${realm}" \
+ "http://polaris:8181/api/management/v1/catalogs"; \
+ -d "${payload}"
+
+echo
+echo "Granting catalog_admin CATALOG_MANAGE_CONTENT on ${catalog_name}..."
+curl --fail-with-body --silent \
+ -H "Authorization: Bearer ${token}" \
+ -H "Content-Type: application/json" \
+ -H "Polaris-Realm: ${realm}" \
+ -X PUT \
+
"http://polaris:8181/api/management/v1/catalogs/${catalog_name}/catalog-roles/catalog_admin/grants";
\
+ -d '{"type":"catalog", "privilege":"CATALOG_MANAGE_CONTENT"}'
+
+echo
+echo "Polaris catalog setup complete."
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/polaris-smoketest.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/polaris-smoketest.sh
new file mode 100755
index 00000000000..a53bb9e9a47
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/polaris-smoketest.sh
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e -u -o pipefail
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+if [[ -z "${RANGER_VERSION:-}" ]]; then
+ # shellcheck source=/dev/null
+ source "${COMPOSE_DIR}/.env"
+fi
+
+# shellcheck source=/dev/null
+source "${COMPOSE_DIR}/../testlib.sh"
+
+: "${POLARIS_IMAGE:=apache/polaris:1.4.1}"
+: "${SPARK_SQL_IMAGE:=apache/spark:3.5.7-scala2.12-java17-ubuntu}"
+: "${POLARIS_CATALOG_NAME:=quickstart_catalog}"
+: "${POLARIS_STORAGE_LOCATION:=s3://iceberg-obs/polaris-smoke}"
+: "${POLARIS_ICEBERG_SPARK_RUNTIME_VERSION:=1.10.1}"
+: "${ICEBERG_SVC_CATALOG_USER:=svc-iceberg-rest-catalog}"
+:
"${ICEBERG_SVC_CATALOG_PRINCIPAL:=${ICEBERG_SVC_CATALOG_USER}/[email protected]}"
+:
"${ICEBERG_SVC_CATALOG_KEYTAB:=/etc/security/keytabs/${ICEBERG_SVC_CATALOG_USER}.keytab}"
+
+export POLARIS_IMAGE SPARK_SQL_IMAGE POLARIS_CATALOG_NAME
POLARIS_STORAGE_LOCATION
+
+if [[ "${COMPOSE_FILE:-}" != *polaris.yaml* ]]; then
+ export
COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.yaml:ranger.yaml:../common/ranger.yaml}:polaris.yaml"
+fi
+
+echo "Fetching permanent S3 credentials for ${ICEBERG_SVC_CATALOG_USER}..."
+s3_secret_output="$(
+ docker-compose exec -T s3g bash -lc \
+ "kinit -kt ${ICEBERG_SVC_CATALOG_KEYTAB} ${ICEBERG_SVC_CATALOG_PRINCIPAL}
&& ozone sh volume info s3v && ozone s3 getsecret"
+)"
+
+POLARIS_AWS_ACCESS_KEY_ID="$(
+ echo "${s3_secret_output}" | grep -o 'awsAccessKey=[^[:space:]]*' | head -1
| cut -d= -f2
+)"
+POLARIS_AWS_SECRET_ACCESS_KEY="$(
+ echo "${s3_secret_output}" | grep -o 'awsSecret=[^[:space:]]*' | head -1 |
cut -d= -f2
+)"
+
+if [[ -z "${POLARIS_AWS_ACCESS_KEY_ID}" || -z
"${POLARIS_AWS_SECRET_ACCESS_KEY}" ]]; then
+ echo "ERROR: Failed to parse S3 credentials from ozone s3 getsecret output:"
+ echo "${s3_secret_output}"
+ exit 1
+fi
+
+export POLARIS_AWS_ACCESS_KEY_ID POLARIS_AWS_SECRET_ACCESS_KEY
+
+echo "Starting Polaris (${POLARIS_IMAGE})..."
+docker-compose --ansi never up -d polaris
+
+wait_for_port polaris 8181 120
+
+echo "Provisioning Polaris catalog (${POLARIS_CATALOG_NAME})..."
+docker-compose --ansi never run --rm polaris-setup
+
+spark_packages="org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:${POLARIS_ICEBERG_SPARK_RUNTIME_VERSION},org.apache.iceberg:iceberg-aws-bundle:${POLARIS_ICEBERG_SPARK_RUNTIME_VERSION}"
+sql_file="/opt/hadoop/smoketest/security/ozone-secure-sts-polaris.sql"
+
+echo "Running Spark SQL against Polaris with STS vended credentials..."
+set +e
+spark_output="$(
+ docker-compose --ansi never run --rm spark-sql \
+ /opt/spark/bin/spark-sql \
+ --packages "${spark_packages}" \
+ --conf
spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
\
+ --conf spark.sql.catalog.polaris=org.apache.iceberg.spark.SparkCatalog \
+ --conf spark.sql.catalog.polaris.type=rest \
+ --conf spark.sql.catalog.polaris.uri=http://polaris:8181/api/catalog \
+ --conf spark.sql.catalog.polaris.rest.auth.type=oauth2 \
+ --conf
spark.sql.catalog.polaris.oauth2-server-uri=http://polaris:8181/api/catalog/v1/oauth/tokens
\
+ --conf spark.sql.catalog.polaris.token-refresh-enabled=false \
+ --conf spark.sql.catalog.polaris.warehouse="${POLARIS_CATALOG_NAME}" \
+ --conf spark.sql.catalog.polaris.scope=PRINCIPAL_ROLE:ALL \
+ --conf spark.sql.catalog.polaris.credential=root:s3cr3t \
+ --conf spark.sql.catalog.polaris.client.region=us-west-2 \
+ --conf
spark.sql.catalog.polaris.header.X-Iceberg-Access-Delegation=vended-credentials
\
+ -f "${sql_file}" 2>&1
+)"
+spark_exit_code=$?
+set -e
+
+echo "${spark_output}"
+
+if [[ "${spark_exit_code}" -ne 0 ]]; then
+ echo "ERROR: spark-sql exited with status ${spark_exit_code}"
+ exit "${spark_exit_code}"
+fi
+
+if ! echo "${spark_output}" | grep -Fq "testing STS"; then
+ echo "ERROR: Expected Spark output to contain inserted row value 'testing
STS'"
+ exit 1
+fi
+
+echo "Polaris STS smoke test passed."
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/polaris.yaml
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/polaris.yaml
new file mode 100644
index 00000000000..cfd6d127c0c
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/polaris.yaml
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Apache Polaris + Spark SQL overlay for ozonesecure-ha STS smoketests.
+# Requires POLARIS_AWS_ACCESS_KEY_ID / POLARIS_AWS_SECRET_ACCESS_KEY at runtime
+# (fetched from ozone s3 getsecret by polaris-smoketest.sh).
+
+services:
+ polaris:
+ image: ${POLARIS_IMAGE}
+ hostname: polaris
+ dns_search: .
+ ports:
+ - 8181:8181
+ environment:
+ AWS_REGION: us-west-2
+ AWS_ACCESS_KEY_ID: ${POLARIS_AWS_ACCESS_KEY_ID}
+ AWS_SECRET_ACCESS_KEY: ${POLARIS_AWS_SECRET_ACCESS_KEY}
+ POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,root,s3cr3t
+ polaris.realm-context.realms: POLARIS
+ polaris.features."ALLOW_SETTING_S3_ENDPOINTS": "true"
+ quarkus.otel.sdk.disabled: "true"
+ healthcheck:
+ test: ["CMD", "curl", "--fail", "http://localhost:8182/q/health";]
+ interval: 2s
+ timeout: 10s
+ retries: 60
+ start_period: 10s
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.124
+
+ polaris-setup:
+ image: alpine/curl:8.19.0
+ dns_search: .
+ depends_on:
+ polaris:
+ condition: service_healthy
+ environment:
+ CLIENT_ID: root
+ CLIENT_SECRET: s3cr3t
+ POLARIS_REALM: POLARIS
+ POLARIS_CATALOG_NAME: ${POLARIS_CATALOG_NAME:-quickstart_catalog}
+ POLARIS_STORAGE_LOCATION:
${POLARIS_STORAGE_LOCATION:-s3://iceberg-obs/polaris-smoke}
+ POLARIS_S3_ENDPOINT: http://s3g:9878
+ POLARIS_STS_ENDPOINT: http://s3g:9880/sts
+ POLARIS_ROLE_ARN:
arn:aws:iam::123456789012:role/iceberg-data-all-access-obs
+ POLARIS_AWS_ACCESS_KEY_ID: ${POLARIS_AWS_ACCESS_KEY_ID}
+ POLARIS_AWS_SECRET_ACCESS_KEY: ${POLARIS_AWS_SECRET_ACCESS_KEY}
+ volumes:
+ - ./polaris-setup.sh:/polaris-setup.sh:ro
+ entrypoint: ["/bin/sh", "/polaris-setup.sh"]
+ networks:
+ ozone_net: {}
+
+ spark-sql:
+ image: ${SPARK_SQL_IMAGE}
+ user: "0:0"
+ dns_search: .
+ volumes:
+ - ../..:/opt/hadoop
+ - ${RANGER_M2_DIR:-${HOME}/.m2}:/root/.m2
+ networks:
+ ozone_net: {}
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-ranger.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-ranger.sh
index 24cc21778f2..4940e782da5 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-ranger.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-ranger.sh
@@ -115,3 +115,4 @@ execute_robot_test s3g freon/validate.robot
execute_robot_test s3g -v RANGER_ENDPOINT_URL:"http://ranger:6080"; -v
USER:hdfs security/ozone-secure-tenant.robot
execute_robot_test s3g -v RANGER_ENDPOINT_URL:"http://ranger:6080"; -v
USER:hdfs security/ozone-secure-sts.robot
execute_robot_test s3g -v RANGER_ENDPOINT_URL:"http://ranger:6080"; -v
USER:hdfs security/ozone-secure-sts-multitenant.robot
+"${COMPOSE_DIR}/polaris-smoketest.sh"
diff --git
a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts-polaris.sql
b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts-polaris.sql
new file mode 100644
index 00000000000..85ae802273b
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts-polaris.sql
@@ -0,0 +1,27 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+USE polaris;
+
+CREATE NAMESPACE IF NOT EXISTS ozone_sts_smoke;
+
+DROP TABLE IF EXISTS ozone_sts_smoke.my_table;
+
+CREATE TABLE ozone_sts_smoke.my_table (id INT, name STRING);
+
+INSERT INTO ozone_sts_smoke.my_table VALUES (1, 'testing STS');
+
+SELECT * FROM ozone_sts_smoke.my_table;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
