This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 34556b6e6c [#9223] test(authz): Add GCP credential vending IT (#9222)
34556b6e6c is described below
commit 34556b6e6c197dc785d4611eb97199e15a473b2c
Author: Bharath Krishna <[email protected]>
AuthorDate: Sun Nov 23 22:22:37 2025 -0800
[#9223] test(authz): Add GCP credential vending IT (#9222)
### What changes were proposed in this pull request?
Add Google Cloud credential vending integration tests
### Why are the changes needed?
Add IT for Google Cloud credential vending
Fix: #9223
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Integration tests
```
gcloud auth login
export GRAVITINO_TEST_CLOUD_IT=true && export
GRAVITINO_GCS_BUCKET=<bucket> && export GRAVITINO_GCS_PATH_PREFIX=<path>&&
./gradlew :iceberg:iceberg-rest-server:test --tests
"*IcebergRESTGCSTokenAuthorizationIT" --info
```
---
... IcebergRESTCloudTokenAuthorizationBaseIT.java} | 168 +++++++---------
.../test/IcebergRESTGCSTokenAuthorizationIT.java | 119 +++++++++++
.../test/IcebergRESTS3TokenAuthorizationIT.java | 224 +--------------------
3 files changed, 196 insertions(+), 315 deletions(-)
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenAuthorizationIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTCloudTokenAuthorizationBaseIT.java
similarity index 69%
copy from
iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenAuthorizationIT.java
copy to
iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTCloudTokenAuthorizationBaseIT.java
index 1f3edd453c..c49cf9e8df 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenAuthorizationIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTCloudTokenAuthorizationBaseIT.java
@@ -20,7 +20,6 @@
package org.apache.gravitino.iceberg.integration.test;
import com.google.common.collect.ImmutableList;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -35,59 +34,24 @@ import org.apache.gravitino.authorization.Owner;
import org.apache.gravitino.authorization.Privileges;
import org.apache.gravitino.authorization.SecurableObject;
import org.apache.gravitino.authorization.SecurableObjects;
-import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
-import org.apache.gravitino.credential.CredentialConstants;
-import org.apache.gravitino.credential.S3TokenCredential;
-import org.apache.gravitino.integration.test.util.BaseIT;
-import org.apache.gravitino.integration.test.util.DownloaderUtils;
import org.apache.gravitino.integration.test.util.ITUtils;
-import org.apache.gravitino.storage.S3Properties;
import org.apache.iceberg.exceptions.ForbiddenException;
import org.apache.spark.SparkException;
import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
-import org.junit.platform.commons.util.StringUtils;
+/**
+ * Abstract base class for Iceberg REST credential vending integration tests
with cloud storage
+ * providers (S3, GCS, Azure, etc.).
+ *
+ * <p>This class contains all common test logic and helper methods for testing
privilege-based
+ * credential vending. Subclasses implement cloud-specific configuration and
setup.
+ */
@SuppressWarnings("FormatStringAnnotation")
-@EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches =
"true")
-public class IcebergRESTS3TokenAuthorizationIT extends IcebergAuthorizationIT {
-
- private static final String SCHEMA_NAME = "schema";
- private String s3Warehouse;
- private String accessKey;
- private String secretKey;
- private String region;
- private String roleArn;
- private String externalId;
-
- @BeforeAll
- public void startIntegrationTest() throws Exception {
- this.s3Warehouse =
- String.format(
- "s3://%s/test1",
System.getenv().getOrDefault("GRAVITINO_S3_BUCKET", "{BUCKET_NAME}"));
- this.accessKey = System.getenv().getOrDefault("GRAVITINO_S3_ACCESS_KEY",
"{ACCESS_KEY}");
- this.secretKey = System.getenv().getOrDefault("GRAVITINO_S3_SECRET_KEY",
"{SECRET_KEY}");
- this.region = System.getenv().getOrDefault("GRAVITINO_S3_REGION",
"ap-southeast-2");
- this.roleArn = System.getenv().getOrDefault("GRAVITINO_S3_ROLE_ARN",
"{ROLE_ARN}");
- this.externalId = System.getenv().getOrDefault("GRAVITINO_S3_EXTERNAL_ID",
"");
-
- super.startIntegrationTest();
-
- catalogClientWithAllPrivilege.asSchemas().createSchema(SCHEMA_NAME,
"test", new HashMap<>());
+public abstract class IcebergRESTCloudTokenAuthorizationBaseIT extends
IcebergAuthorizationIT {
- if (ITUtils.isEmbedded()) {
- return;
- }
- try {
- downloadIcebergAwsBundleJar();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- copyS3BundleJar();
- }
+ protected static final String SCHEMA_NAME = "schema";
@BeforeEach
void revokePrivilege() {
@@ -105,20 +69,63 @@ public class IcebergRESTS3TokenAuthorizationIT extends
IcebergAuthorizationIT {
}
@Override
- public Map<String, String> getCustomProperties() {
- HashMap<String, String> m = new HashMap<>();
- m.putAll(getS3Config());
- return m;
+ protected boolean supportsCredentialVending() {
+ return true;
}
+ /**
+ * Returns the cloud-specific configuration properties. Subclasses implement
this to provide
+ * cloud-specific settings like credentials, warehouse location, IO
implementation, etc.
+ *
+ * @return Map of cloud storage configuration properties
+ */
@Override
- protected boolean supportsCredentialVending() {
- return true;
+ public abstract Map<String, String> getCustomProperties();
+
+ /**
+ * Downloads cloud-specific bundle JARs (e.g., iceberg-aws-bundle,
iceberg-gcp-bundle). Subclasses
+ * implement this to download the appropriate bundle for their cloud
provider.
+ *
+ * @throws Exception if download fails
+ */
+ protected abstract void downloadCloudBundleJar() throws Exception;
+
+ /**
+ * Copies cloud-specific bundle JARs to the Iceberg REST server libs
directory. Subclasses
+ * implement this to copy the appropriate bundle (e.g., "aws", "gcp",
"azure").
+ */
+ protected abstract void copyCloudBundleJar();
+
+ /**
+ * Returns the cloud provider name for table naming (e.g., "s3", "gcs",
"azure"). Used to generate
+ * unique table names like "test_owner_s3", "test_select_gcs".
+ *
+ * @return Cloud provider suffix for table names
+ */
+ protected abstract String getCloudProviderName();
+
+ /**
+ * Sets up cloud-specific bundle JARs by downloading and copying them. This
method should be
+ * called from subclass {@code startIntegrationTest()} methods after calling
{@code
+ * super.startIntegrationTest()}.
+ *
+ * <p>Skips setup if running in embedded mode.
+ */
+ protected void setupCloudBundles() {
+ if (ITUtils.isEmbedded()) {
+ return;
+ }
+ try {
+ downloadCloudBundleJar();
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to download cloud bundle JAR", e);
+ }
+ copyCloudBundleJar();
}
@Test
- void testIcebergOwnerS3Token() {
- String tableName = "test_owner_s3";
+ void testIcebergOwnerCloudToken() {
+ String tableName = "test_owner_" + getCloudProviderName();
grantCreateTableRole(SCHEMA_NAME);
sql("CREATE TABLE %s(a int, b int) PARTITIONED BY (a)", tableName);
sql("INSERT INTO %s VALUES (1,1),(2,2)", tableName);
@@ -139,8 +146,8 @@ public class IcebergRESTS3TokenAuthorizationIT extends
IcebergAuthorizationIT {
}
@Test
- void testIcebergSelectTableS3Token() {
- String tableName = "test_select_s3";
+ void testIcebergSelectTableCloudToken() {
+ String tableName = "test_select_" + getCloudProviderName();
createTable(SCHEMA_NAME, tableName);
// No privileges
@@ -170,8 +177,8 @@ public class IcebergRESTS3TokenAuthorizationIT extends
IcebergAuthorizationIT {
}
@Test
- void testIcebergModifyTableS3Token() {
- String tableName = "test_modify_s3";
+ void testIcebergModifyTableCloudToken() {
+ String tableName = "test_modify_" + getCloudProviderName();
createTable(SCHEMA_NAME, tableName);
// No privileges
@@ -182,7 +189,6 @@ public class IcebergRESTS3TokenAuthorizationIT extends
IcebergAuthorizationIT {
() -> sql("SELECT * FROM %s.%s.%s", SPARK_CATALOG_NAME, SCHEMA_NAME,
tableName));
grantModifyTableRole(tableName);
-
sql("INSERT INTO %s VALUES (1,1),(2,2)", tableName);
List<Object[]> rows = sql("SELECT * FROM %s", tableName);
Assertions.assertEquals(2, rows.size());
@@ -200,7 +206,7 @@ public class IcebergRESTS3TokenAuthorizationIT extends
IcebergAuthorizationIT {
Assertions.assertEquals(2, rows.size());
}
- private void grantUseSchemaRole(String schema) {
+ protected void grantUseSchemaRole(String schema) {
String roleName = "useSchema_" + UUID.randomUUID();
List<SecurableObject> securableObjects = new ArrayList<>();
SecurableObject catalogObject =
@@ -216,7 +222,7 @@ public class IcebergRESTS3TokenAuthorizationIT extends
IcebergAuthorizationIT {
metalakeClientWithAllPrivilege.grantRolesToUser(ImmutableList.of(roleName),
NORMAL_USER);
}
- private String grantCreateTableRole(String schema) {
+ protected String grantCreateTableRole(String schema) {
String roleName = "createTable_" + UUID.randomUUID();
List<SecurableObject> securableObjects = new ArrayList<>();
SecurableObject catalogObject =
@@ -232,7 +238,7 @@ public class IcebergRESTS3TokenAuthorizationIT extends
IcebergAuthorizationIT {
return roleName;
}
- private String grantSelectTableRole(String tableName) {
+ protected String grantSelectTableRole(String tableName) {
String roleName = "selectTable_" + UUID.randomUUID();
List<SecurableObject> securableObjects = new ArrayList<>();
SecurableObject catalogObject =
@@ -251,7 +257,7 @@ public class IcebergRESTS3TokenAuthorizationIT extends
IcebergAuthorizationIT {
return roleName;
}
- private String grantModifyTableRole(String tableName) {
+ protected String grantModifyTableRole(String tableName) {
String roleName = "modifyTable_" + UUID.randomUUID();
List<SecurableObject> securableObjects = new ArrayList<>();
SecurableObject catalogObject =
@@ -270,7 +276,7 @@ public class IcebergRESTS3TokenAuthorizationIT extends
IcebergAuthorizationIT {
return roleName;
}
- private void clearTable() {
+ protected void clearTable() {
Arrays.stream(
catalogClientWithAllPrivilege.asTableCatalog().listTables(Namespace.of(SCHEMA_NAME)))
.forEach(
@@ -283,40 +289,4 @@ public class IcebergRESTS3TokenAuthorizationIT extends
IcebergAuthorizationIT {
catalogClientWithAllPrivilege.asTableCatalog().listTables(Namespace.of(SCHEMA_NAME));
Assertions.assertEquals(0, nameIdentifiers.length);
}
-
- private void downloadIcebergAwsBundleJar() throws IOException {
- String icebergBundleJarUri =
- String.format(
- "https://repo1.maven.org/maven2/org/apache/iceberg/";
- + "iceberg-aws-bundle/%s/iceberg-aws-bundle-%s.jar",
- ITUtils.icebergVersion(), ITUtils.icebergVersion());
- String gravitinoHome = System.getenv("GRAVITINO_HOME");
- String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
- DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);
- }
-
- private void copyS3BundleJar() {
- String gravitinoHome = System.getenv("GRAVITINO_HOME");
- String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
- BaseIT.copyBundleJarsToDirectory("aws", targetDir);
- }
-
- private Map<String, String> getS3Config() {
- Map configMap = new HashMap<String, String>();
-
- configMap.put(
- CredentialConstants.CREDENTIAL_PROVIDERS,
S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE);
- configMap.put(S3Properties.GRAVITINO_S3_REGION, region);
- configMap.put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, accessKey);
- configMap.put(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, secretKey);
- configMap.put(S3Properties.GRAVITINO_S3_ROLE_ARN, roleArn);
- if (StringUtils.isNotBlank(externalId)) {
- configMap.put(S3Properties.GRAVITINO_S3_EXTERNAL_ID, externalId);
- }
-
- configMap.put(IcebergConstants.IO_IMPL,
"org.apache.iceberg.aws.s3.S3FileIO");
- configMap.put(IcebergConstants.WAREHOUSE, s3Warehouse);
-
- return configMap;
- }
}
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSTokenAuthorizationIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSTokenAuthorizationIT.java
new file mode 100644
index 0000000000..00b625b0d9
--- /dev/null
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSTokenAuthorizationIT.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.iceberg.integration.test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
+import org.apache.gravitino.credential.CredentialConstants;
+import org.apache.gravitino.credential.GCSTokenCredential;
+import org.apache.gravitino.integration.test.util.BaseIT;
+import org.apache.gravitino.integration.test.util.DownloaderUtils;
+import org.apache.gravitino.integration.test.util.ITUtils;
+import org.apache.gravitino.storage.GCSProperties;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
+
+/**
+ * Integration tests for Iceberg REST credential vending with GCS (Google
Cloud Storage).
+ *
+ * <p>Authentication: Supports both service account key file and Application
Default Credentials
+ * (ADC).
+ *
+ * <p>Required environment variables:
+ *
+ * <ul>
+ * <li>GRAVITINO_TEST_CLOUD_IT=true - enables cloud integration tests
+ * <li>GRAVITINO_GCS_BUCKET - GCS bucket name (e.g., "my-bucket")
+ * <li>GRAVITINO_GCS_PATH_PREFIX - path prefix within bucket (e.g.,
"test/gravitino")
+ * <li>GRAVITINO_GCS_SERVICE_ACCOUNT_FILE (optional) - path to service
account JSON key. If
+ * omitted, uses ADC (requires: gcloud auth application-default login)
+ * </ul>
+ */
+@SuppressWarnings("FormatStringAnnotation")
+@EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches =
"true")
+public class IcebergRESTGCSTokenAuthorizationIT extends
IcebergRESTCloudTokenAuthorizationBaseIT {
+
+ private String gcsWarehouse;
+ private String serviceAccountFile;
+
+ @BeforeAll
+ public void startIntegrationTest() throws Exception {
+ String bucket = System.getenv().getOrDefault("GRAVITINO_GCS_BUCKET",
"{BUCKET_NAME}");
+ String pathPrefix =
System.getenv().getOrDefault("GRAVITINO_GCS_PATH_PREFIX", "test1");
+ this.gcsWarehouse = String.format("gs://%s/%s", bucket, pathPrefix);
+ // Use null to trigger ADC (Application Default Credentials) if not
explicitly provided
+ this.serviceAccountFile =
System.getenv().get("GRAVITINO_GCS_SERVICE_ACCOUNT_FILE");
+
+ super.startIntegrationTest();
+
+ catalogClientWithAllPrivilege.asSchemas().createSchema(SCHEMA_NAME,
"test", new HashMap<>());
+
+ setupCloudBundles();
+ }
+
+ @Override
+ public Map<String, String> getCustomProperties() {
+ HashMap<String, String> m = new HashMap<>();
+ m.putAll(getGCSConfig());
+ return m;
+ }
+
+ @Override
+ protected String getCloudProviderName() {
+ return "gcs";
+ }
+
+ @Override
+ protected void downloadCloudBundleJar() throws IOException {
+ String icebergBundleJarUri =
+ String.format(
+ "https://repo1.maven.org/maven2/org/apache/iceberg/";
+ + "iceberg-gcp-bundle/%s/iceberg-gcp-bundle-%s.jar",
+ ITUtils.icebergVersion(), ITUtils.icebergVersion());
+ String gravitinoHome = System.getenv("GRAVITINO_HOME");
+ String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
+ DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);
+ }
+
+ @Override
+ protected void copyCloudBundleJar() {
+ String gravitinoHome = System.getenv("GRAVITINO_HOME");
+ String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
+ BaseIT.copyBundleJarsToDirectory("gcp", targetDir);
+ }
+
+ private Map<String, String> getGCSConfig() {
+ Map<String, String> configMap = new HashMap<>();
+
+ configMap.put(
+ CredentialConstants.CREDENTIAL_PROVIDERS,
GCSTokenCredential.GCS_TOKEN_CREDENTIAL_TYPE);
+ // Only set service account file if explicitly provided, otherwise use ADC
+ if (serviceAccountFile != null && !serviceAccountFile.isEmpty()) {
+ configMap.put(GCSProperties.GRAVITINO_GCS_SERVICE_ACCOUNT_FILE,
serviceAccountFile);
+ }
+
+ configMap.put(IcebergConstants.IO_IMPL,
"org.apache.iceberg.gcp.gcs.GCSFileIO");
+ configMap.put(IcebergConstants.WAREHOUSE, gcsWarehouse);
+
+ return configMap;
+ }
+}
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenAuthorizationIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenAuthorizationIT.java
index 1f3edd453c..ac5c85729b 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenAuthorizationIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenAuthorizationIT.java
@@ -19,22 +19,9 @@
package org.apache.gravitino.iceberg.integration.test;
-import com.google.common.collect.ImmutableList;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
-import java.util.UUID;
-import org.apache.gravitino.MetadataObject;
-import org.apache.gravitino.MetadataObjects;
-import org.apache.gravitino.NameIdentifier;
-import org.apache.gravitino.Namespace;
-import org.apache.gravitino.authorization.Owner;
-import org.apache.gravitino.authorization.Privileges;
-import org.apache.gravitino.authorization.SecurableObject;
-import org.apache.gravitino.authorization.SecurableObjects;
import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
import org.apache.gravitino.credential.CredentialConstants;
import org.apache.gravitino.credential.S3TokenCredential;
@@ -42,20 +29,14 @@ import org.apache.gravitino.integration.test.util.BaseIT;
import org.apache.gravitino.integration.test.util.DownloaderUtils;
import org.apache.gravitino.integration.test.util.ITUtils;
import org.apache.gravitino.storage.S3Properties;
-import org.apache.iceberg.exceptions.ForbiddenException;
-import org.apache.spark.SparkException;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.junit.platform.commons.util.StringUtils;
@SuppressWarnings("FormatStringAnnotation")
@EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches =
"true")
-public class IcebergRESTS3TokenAuthorizationIT extends IcebergAuthorizationIT {
+public class IcebergRESTS3TokenAuthorizationIT extends
IcebergRESTCloudTokenAuthorizationBaseIT {
- private static final String SCHEMA_NAME = "schema";
private String s3Warehouse;
private String accessKey;
private String secretKey;
@@ -78,30 +59,7 @@ public class IcebergRESTS3TokenAuthorizationIT extends
IcebergAuthorizationIT {
catalogClientWithAllPrivilege.asSchemas().createSchema(SCHEMA_NAME,
"test", new HashMap<>());
- if (ITUtils.isEmbedded()) {
- return;
- }
- try {
- downloadIcebergAwsBundleJar();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- copyS3BundleJar();
- }
-
- @BeforeEach
- void revokePrivilege() {
- revokeUserRoles();
- resetMetalakeAndCatalogOwner();
- MetadataObject schemaObject =
- MetadataObjects.of(
- Arrays.asList(GRAVITINO_CATALOG_NAME, SCHEMA_NAME),
MetadataObject.Type.SCHEMA);
- metalakeClientWithAllPrivilege.setOwner(schemaObject, SUPER_USER,
Owner.Type.USER);
- clearTable();
- // Grant user the privilege to use the catalog and schema
- grantUseSchemaRole(SCHEMA_NAME);
- sql("USE %s;", SPARK_CATALOG_NAME);
- sql("USE %s;", SCHEMA_NAME);
+ setupCloudBundles();
}
@Override
@@ -112,179 +70,12 @@ public class IcebergRESTS3TokenAuthorizationIT extends
IcebergAuthorizationIT {
}
@Override
- protected boolean supportsCredentialVending() {
- return true;
- }
-
- @Test
- void testIcebergOwnerS3Token() {
- String tableName = "test_owner_s3";
- grantCreateTableRole(SCHEMA_NAME);
- sql("CREATE TABLE %s(a int, b int) PARTITIONED BY (a)", tableName);
- sql("INSERT INTO %s VALUES (1,1),(2,2)", tableName);
- List<Object[]> rows = sql("SELECT * FROM %s", tableName);
- Assertions.assertEquals(2, rows.size());
-
- rows = sql("SELECT * FROM %s.%s.%s", SPARK_CATALOG_NAME, SCHEMA_NAME,
tableName);
- Assertions.assertEquals(2, rows.size());
-
- rows = sql("SELECT * FROM %s.%s", SCHEMA_NAME, tableName);
- Assertions.assertEquals(2, rows.size());
-
- rows = sql("SELECT * FROM %s.%s.%s.partitions", SPARK_CATALOG_NAME,
SCHEMA_NAME, tableName);
- Assertions.assertEquals(2, rows.size());
-
- rows = sql("SELECT *,_file FROM %s", tableName);
- Assertions.assertEquals(2, rows.size());
- }
-
- @Test
- void testIcebergSelectTableS3Token() {
- String tableName = "test_select_s3";
- createTable(SCHEMA_NAME, tableName);
-
- // No privileges
- Assertions.assertThrows(
- ForbiddenException.class, () -> sql("INSERT INTO %s VALUES
(1,1),(2,2)", tableName));
- Assertions.assertThrows(
- ForbiddenException.class,
- () -> sql("SELECT * FROM %s.%s.%s", SPARK_CATALOG_NAME, SCHEMA_NAME,
tableName));
-
- grantSelectTableRole(tableName);
- Assertions.assertThrows(
- SparkException.class, () -> sql("INSERT INTO %s VALUES (1,1),(2,2)",
tableName));
- List<Object[]> rows = sql("SELECT * FROM %s", tableName);
- Assertions.assertEquals(0, rows.size());
-
- rows = sql("SELECT * FROM %s.%s.%s", SPARK_CATALOG_NAME, SCHEMA_NAME,
tableName);
- Assertions.assertEquals(0, rows.size());
-
- rows = sql("SELECT * FROM %s.%s", SCHEMA_NAME, tableName);
- Assertions.assertEquals(0, rows.size());
-
- rows = sql("SELECT * FROM %s.%s.%s.partitions", SPARK_CATALOG_NAME,
SCHEMA_NAME, tableName);
- Assertions.assertEquals(0, rows.size());
-
- rows = sql("SELECT *,_file FROM %s", tableName);
- Assertions.assertEquals(0, rows.size());
- }
-
- @Test
- void testIcebergModifyTableS3Token() {
- String tableName = "test_modify_s3";
- createTable(SCHEMA_NAME, tableName);
-
- // No privileges
- Assertions.assertThrows(
- ForbiddenException.class, () -> sql("INSERT INTO %s VALUES
(1,1),(2,2)", tableName));
- Assertions.assertThrows(
- ForbiddenException.class,
- () -> sql("SELECT * FROM %s.%s.%s", SPARK_CATALOG_NAME, SCHEMA_NAME,
tableName));
-
- grantModifyTableRole(tableName);
-
- sql("INSERT INTO %s VALUES (1,1),(2,2)", tableName);
- List<Object[]> rows = sql("SELECT * FROM %s", tableName);
- Assertions.assertEquals(2, rows.size());
-
- rows = sql("SELECT * FROM %s.%s.%s", SPARK_CATALOG_NAME, SCHEMA_NAME,
tableName);
- Assertions.assertEquals(2, rows.size());
-
- rows = sql("SELECT * FROM %s.%s", SCHEMA_NAME, tableName);
- Assertions.assertEquals(2, rows.size());
-
- rows = sql("SELECT * FROM %s.%s.%s.partitions", SPARK_CATALOG_NAME,
SCHEMA_NAME, tableName);
- Assertions.assertEquals(1, rows.size());
-
- rows = sql("SELECT *,_file FROM %s", tableName);
- Assertions.assertEquals(2, rows.size());
- }
-
- private void grantUseSchemaRole(String schema) {
- String roleName = "useSchema_" + UUID.randomUUID();
- List<SecurableObject> securableObjects = new ArrayList<>();
- SecurableObject catalogObject =
- SecurableObjects.ofCatalog(
- GRAVITINO_CATALOG_NAME,
ImmutableList.of(Privileges.UseCatalog.allow()));
- securableObjects.add(catalogObject);
- SecurableObject schemaObject =
- SecurableObjects.ofSchema(
- catalogObject, schema,
ImmutableList.of(Privileges.UseSchema.allow()));
- securableObjects.add(schemaObject);
- metalakeClientWithAllPrivilege.createRole(roleName, new HashMap<>(),
securableObjects);
-
-
metalakeClientWithAllPrivilege.grantRolesToUser(ImmutableList.of(roleName),
NORMAL_USER);
+ protected String getCloudProviderName() {
+ return "s3";
}
- private String grantCreateTableRole(String schema) {
- String roleName = "createTable_" + UUID.randomUUID();
- List<SecurableObject> securableObjects = new ArrayList<>();
- SecurableObject catalogObject =
- SecurableObjects.ofCatalog(
- GRAVITINO_CATALOG_NAME,
ImmutableList.of(Privileges.UseCatalog.allow()));
- securableObjects.add(catalogObject);
- SecurableObject schemaObject =
- SecurableObjects.ofSchema(
- catalogObject, schema,
ImmutableList.of(Privileges.CreateTable.allow()));
- securableObjects.add(schemaObject);
- metalakeClientWithAllPrivilege.createRole(roleName, new HashMap<>(),
securableObjects);
-
metalakeClientWithAllPrivilege.grantRolesToUser(ImmutableList.of(roleName),
NORMAL_USER);
- return roleName;
- }
-
- private String grantSelectTableRole(String tableName) {
- String roleName = "selectTable_" + UUID.randomUUID();
- List<SecurableObject> securableObjects = new ArrayList<>();
- SecurableObject catalogObject =
- SecurableObjects.ofCatalog(
- GRAVITINO_CATALOG_NAME,
ImmutableList.of(Privileges.UseCatalog.allow()));
- securableObjects.add(catalogObject);
- SecurableObject schemaObject =
- SecurableObjects.ofSchema(
- catalogObject, SCHEMA_NAME,
ImmutableList.of(Privileges.UseSchema.allow()));
- SecurableObject tableObject =
- SecurableObjects.ofTable(
- schemaObject, tableName,
ImmutableList.of(Privileges.SelectTable.allow()));
- securableObjects.add(tableObject);
- metalakeClientWithAllPrivilege.createRole(roleName, new HashMap<>(),
securableObjects);
-
metalakeClientWithAllPrivilege.grantRolesToUser(ImmutableList.of(roleName),
NORMAL_USER);
- return roleName;
- }
-
- private String grantModifyTableRole(String tableName) {
- String roleName = "modifyTable_" + UUID.randomUUID();
- List<SecurableObject> securableObjects = new ArrayList<>();
- SecurableObject catalogObject =
- SecurableObjects.ofCatalog(
- GRAVITINO_CATALOG_NAME,
ImmutableList.of(Privileges.UseCatalog.allow()));
- securableObjects.add(catalogObject);
- SecurableObject schemaObject =
- SecurableObjects.ofSchema(
- catalogObject, SCHEMA_NAME,
ImmutableList.of(Privileges.UseSchema.allow()));
- SecurableObject tableObject =
- SecurableObjects.ofTable(
- schemaObject, tableName,
ImmutableList.of(Privileges.ModifyTable.allow()));
- securableObjects.add(tableObject);
- metalakeClientWithAllPrivilege.createRole(roleName, new HashMap<>(),
securableObjects);
-
metalakeClientWithAllPrivilege.grantRolesToUser(ImmutableList.of(roleName),
NORMAL_USER);
- return roleName;
- }
-
- private void clearTable() {
- Arrays.stream(
-
catalogClientWithAllPrivilege.asTableCatalog().listTables(Namespace.of(SCHEMA_NAME)))
- .forEach(
- table -> {
- catalogClientWithAllPrivilege
- .asTableCatalog()
- .dropTable(NameIdentifier.of(SCHEMA_NAME, table.name()));
- });
- NameIdentifier[] nameIdentifiers =
-
catalogClientWithAllPrivilege.asTableCatalog().listTables(Namespace.of(SCHEMA_NAME));
- Assertions.assertEquals(0, nameIdentifiers.length);
- }
-
- private void downloadIcebergAwsBundleJar() throws IOException {
+ @Override
+ protected void downloadCloudBundleJar() throws IOException {
String icebergBundleJarUri =
String.format(
"https://repo1.maven.org/maven2/org/apache/iceberg/";
@@ -295,7 +86,8 @@ public class IcebergRESTS3TokenAuthorizationIT extends
IcebergAuthorizationIT {
DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);
}
- private void copyS3BundleJar() {
+ @Override
+ protected void copyCloudBundleJar() {
String gravitinoHome = System.getenv("GRAVITINO_HOME");
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
BaseIT.copyBundleJarsToDirectory("aws", targetDir);
