This is an automated email from the ASF dual-hosted git repository.
difin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ad55d58eadd HIVE-29234: Iceberg: Validate HMS REST Catalog Client with
OAuth2 (#6124)
ad55d58eadd is described below
commit ad55d58eadd6c1aabc7fed51e25dfffe158a44f6
Author: Dmitriy Fingerman <[email protected]>
AuthorDate: Mon Oct 13 13:49:21 2025 -0400
HIVE-29234: Iceberg: Validate HMS REST Catalog Client with OAuth2 (#6124)
---
itests/hive-iceberg/pom.xml | 6 ++
...T.java => TestHiveRESTCatalogClientITBase.java} | 95 ++++++++++++----------
.../hive/TestHiveRESTCatalogClientITNoAuth.java | 38 +++++++++
.../hive/TestHiveRESTCatalogClientITOauth2.java | 48 +++++++++++
itests/qtest-iceberg/pom.xml | 31 +++++++
.../hive/cli/HiveRESTCatalogServerExtension.java | 51 ++++++++++--
...bergRESTCatalogGravitinoLlapLocalCliDriver.java | 43 +++++++++-
...estIcebergRESTCatalogHMSLlapLocalCliDriver.java | 38 +++------
.../test/resources/gravitino-h2-test-template.conf | 18 +++-
pom.xml | 2 +
.../rest/extension/OAuth2AuthorizationServer.java | 35 ++++++--
11 files changed, 317 insertions(+), 88 deletions(-)
diff --git a/itests/hive-iceberg/pom.xml b/itests/hive-iceberg/pom.xml
index 170a6c89abd..1ad24643776 100644
--- a/itests/hive-iceberg/pom.xml
+++ b/itests/hive-iceberg/pom.xml
@@ -46,6 +46,12 @@
<classifier>tests</classifier>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.keycloak</groupId>
+ <artifactId>keycloak-admin-client</artifactId>
+ <version>${keycloak.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-standalone-metastore-common</artifactId>
diff --git
a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientIT.java
b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java
similarity index 79%
rename from
itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientIT.java
rename to
itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java
index db5329c6431..788850c7b24 100644
---
a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientIT.java
+++
b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java
@@ -23,9 +23,9 @@
import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.ServletSecurity;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.GetTableRequest;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
@@ -46,59 +46,54 @@
import org.apache.iceberg.hive.CatalogUtils;
import org.apache.iceberg.hive.HiveSchemaUtil;
import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
-import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestInstance;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-import java.util.Collections;
-import java.util.Map;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertThrows;
/*
- * This test is an integration test for the hive-iceberg REST Catalog client
and HMS REST Catalog Server.
- * It uses the HiveMetaStoreClient backed by hive-iceberg REST catalog
adapter to connect to the HMS RESTCatalog Server.
+ * This is an integration test for the HiveMetaStoreClient and HMS REST
Catalog Server. It creates and uses the
+ * HMS IMetaStoreClient backed by HiveMetaStoreClient adapter to connect to
the HMS RESTCatalog Server.
* The flow is as follows:
* Hive ql wrapper --> HiveMetaStoreClient --> HiveRESTCatalogClient --> HMS
RESTCatalog Server --> HMS
*/
-@TestInstance(TestInstance.Lifecycle.PER_CLASS)
-public class TestHiveRESTCatalogClientIT {
-
- private static final String DB_NAME = "ice_db";
- private static final String TABLE_NAME = "ice_tbl";
- private static final String CATALOG_NAME = "ice01";
- private static final String HIVE_ICEBERG_STORAGE_HANDLER =
"org.apache.iceberg.mr.hive.HiveIcebergStorageHandler";
+public abstract class TestHiveRESTCatalogClientITBase {
+
+ static final String DB_NAME = "ice_db";
+ static final String TABLE_NAME = "ice_tbl";
+ static final String CATALOG_NAME = "ice01";
+ static final String HIVE_ICEBERG_STORAGE_HANDLER =
"org.apache.iceberg.mr.hive.HiveIcebergStorageHandler";
+ static final String REST_CATALOG_PREFIX = String.format("%s%s.",
CatalogUtils.CATALOG_CONFIG_PREFIX, CATALOG_NAME);
+
+ HiveConf hiveConf;
+ Configuration conf;
+ Hive hive;
+ IMetaStoreClient msClient;
- private Configuration conf;
- private HiveConf hiveConf;
- private Hive hive;
-
- private IMetaStoreClient msClient;
-
- @RegisterExtension
- private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
- HiveRESTCatalogServerExtension.builder(ServletSecurity.AuthType.NONE)
- .addMetaStoreSchemaClassName(ITestsSchemaInfo.class)
- .build();
+ abstract HiveRESTCatalogServerExtension getHiveRESTCatalogServerExtension();
- @BeforeAll
- public void setup() throws Exception {
- // Starting msClient with Iceberg REST Catalog client underneath
- String restCatalogPrefix = String.format("%s%s.",
CatalogUtils.CATALOG_CONFIG_PREFIX, CATALOG_NAME);
+ public void setupConf() {
+ HiveRESTCatalogServerExtension restCatalogExtension =
getHiveRESTCatalogServerExtension();
- conf = REST_CATALOG_EXTENSION.getConf();
+ conf = restCatalogExtension.getConf();
MetastoreConf.setVar(conf, MetastoreConf.ConfVars.METASTORE_CLIENT_IMPL,
"org.apache.iceberg.hive.client.HiveRESTCatalogClient");
conf.set(MetastoreConf.ConfVars.CATALOG_DEFAULT.getVarname(),
CATALOG_NAME);
- conf.set(restCatalogPrefix + "uri",
REST_CATALOG_EXTENSION.getRestEndpoint());
- conf.set(restCatalogPrefix + "type",
CatalogUtil.ICEBERG_CATALOG_TYPE_REST);
+ conf.set(REST_CATALOG_PREFIX + "uri",
restCatalogExtension.getRestEndpoint());
+ conf.set(REST_CATALOG_PREFIX + "type",
CatalogUtil.ICEBERG_CATALOG_TYPE_REST);
+ }
+
+ @BeforeEach
+ void setup() throws Exception {
+ setupConf();
HiveMetaHookLoader hookLoader = tbl -> {
HiveStorageHandler storageHandler;
@@ -109,18 +104,19 @@ public void setup() throws Exception {
}
return storageHandler == null ? null : storageHandler.getMetaHook();
};
-
+
msClient = new HiveMetaStoreClient(conf, hookLoader);
hiveConf = new HiveConf(conf, HiveConf.class);
hive = Hive.get(hiveConf);
}
- @AfterAll public void tearDown() {
+ @AfterEach
+ public void tearDown() {
if (msClient != null) {
msClient.close();
}
}
-
+
@Test
public void testIceberg() throws Exception {
@@ -142,7 +138,7 @@ public void testIceberg() throws Exception {
// --- Get Databases ---
List<String> dbs = msClient.getDatabases(CATALOG_NAME, "ice_*");
Assertions.assertEquals(1, dbs.size());
- Assertions.assertEquals(DB_NAME, dbs.get(0));
+ Assertions.assertEquals(DB_NAME, dbs.getFirst());
// --- Get All Databases ---
List<String> allDbs = msClient.getAllDatabases(CATALOG_NAME);
@@ -151,7 +147,7 @@ public void testIceberg() throws Exception {
Assertions.assertTrue(allDbs.contains(DB_NAME));
// --- Create Table ---
- org.apache.hadoop.hive.metastore.api.Table tTable =
createPartitionedTable(msClient,
+ Table tTable = createPartitionedTable(msClient,
CATALOG_NAME, DB_NAME, TABLE_NAME, new java.util.HashMap<>());
Assertions.assertNotNull(tTable);
Assertions.assertEquals(HiveMetaHook.ICEBERG,
tTable.getParameters().get(HiveMetaHook.TABLE_TYPE));
@@ -166,7 +162,12 @@ public void testIceberg() throws Exception {
Assertions.assertTrue(msClient.tableExists(CATALOG_NAME, DB_NAME,
TABLE_NAME));
// --- Get Table ---
- org.apache.hadoop.hive.metastore.api.Table table =
msClient.getTable(CATALOG_NAME, DB_NAME, TABLE_NAME);
+ GetTableRequest getTableRequest = new GetTableRequest();
+ getTableRequest.setCatName(CATALOG_NAME);
+ getTableRequest.setDbName(DB_NAME);
+ getTableRequest.setTblName(TABLE_NAME);
+
+ Table table = msClient.getTable(getTableRequest);
Assertions.assertEquals(DB_NAME, table.getDbName());
Assertions.assertEquals(TABLE_NAME, table.getTableName());
Assertions.assertEquals(HIVE_ICEBERG_STORAGE_HANDLER,
table.getParameters().get("storage_handler"));
@@ -193,8 +194,8 @@ public void testIceberg() throws Exception {
Assertions.assertFalse(msClient.getAllDatabases(CATALOG_NAME).contains(DB_NAME));
}
- private static Table createPartitionedTable(IMetaStoreClient db, String
catName, String dbName, String tableName,
- Map<String, String> tableParameters) throws Exception {
+ private static Table createPartitionedTable(IMetaStoreClient db, String
catName, String dbName, String tableName,
+ Map<String, String> tableParameters) throws Exception {
db.dropTable(catName, dbName, tableName);
Table table = new Table();
table.setCatName(catName);
@@ -222,6 +223,12 @@ private static Table
createPartitionedTable(IMetaStoreClient db, String catName,
table.getParameters().put(TableProperties.DEFAULT_PARTITION_SPEC,
specString);
db.createTable(table);
- return db.getTable(catName, dbName, tableName);
+
+ GetTableRequest getTableRequest = new GetTableRequest();
+ getTableRequest.setCatName(CATALOG_NAME);
+ getTableRequest.setDbName(DB_NAME);
+ getTableRequest.setTblName(TABLE_NAME);
+
+ return db.getTable(getTableRequest);
}
}
diff --git
a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITNoAuth.java
b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITNoAuth.java
new file mode 100644
index 00000000000..eaec8243a82
--- /dev/null
+++
b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITNoAuth.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive;
+
+import org.apache.hadoop.hive.metastore.ServletSecurity;
+import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class TestHiveRESTCatalogClientITNoAuth extends
TestHiveRESTCatalogClientITBase {
+
+ @RegisterExtension
+ private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
+ HiveRESTCatalogServerExtension.builder(ServletSecurity.AuthType.NONE)
+ .addMetaStoreSchemaClassName(ITestsSchemaInfo.class)
+ .build();
+
+ @Override
+ HiveRESTCatalogServerExtension getHiveRESTCatalogServerExtension() {
+ return REST_CATALOG_EXTENSION;
+ }
+}
diff --git
a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITOauth2.java
b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITOauth2.java
new file mode 100644
index 00000000000..d2c483ae769
--- /dev/null
+++
b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITOauth2.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive;
+
+import org.apache.hadoop.hive.metastore.ServletSecurity;
+import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class TestHiveRESTCatalogClientITOauth2 extends
TestHiveRESTCatalogClientITBase {
+
+ @RegisterExtension
+ private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
+ HiveRESTCatalogServerExtension.builder(ServletSecurity.AuthType.OAUTH2)
+ .addMetaStoreSchemaClassName(ITestsSchemaInfo.class)
+ .build();
+
+ @Override
+ public void setupConf() {
+ super.setupConf();
+
+ // Oauth2 properties
+ conf.set(REST_CATALOG_PREFIX + "rest.auth.type", "oauth2");
+ conf.set(REST_CATALOG_PREFIX + "oauth2-server-uri",
REST_CATALOG_EXTENSION.getOAuth2TokenEndpoint());
+ conf.set(REST_CATALOG_PREFIX + "credential",
REST_CATALOG_EXTENSION.getOAuth2ClientCredential());
+ }
+
+ @Override
+ HiveRESTCatalogServerExtension getHiveRESTCatalogServerExtension() {
+ return REST_CATALOG_EXTENSION;
+ }
+}
diff --git a/itests/qtest-iceberg/pom.xml b/itests/qtest-iceberg/pom.xml
index c7cd70d7479..9099ba38a6d 100644
--- a/itests/qtest-iceberg/pom.xml
+++ b/itests/qtest-iceberg/pom.xml
@@ -480,6 +480,37 @@
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.keycloak</groupId>
+ <artifactId>keycloak-admin-client</artifactId>
+ <version>${keycloak.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>jakarta.annotation</groupId>
+ <artifactId>jakarta.annotation-api</artifactId>
+ <version>2.1.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.nimbusds</groupId>
+ <artifactId>oauth2-oidc-sdk</artifactId>
+ <version>${nimbus-oauth.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>jakarta.xml.bind</groupId>
+ <artifactId>jakarta.xml.bind-api</artifactId>
+ <version>4.0.4</version>
+ </dependency>
+ <dependency>
+ <groupId>jakarta.activation</groupId>
+ <artifactId>jakarta.activation-api</artifactId>
+ <version>2.1.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jaxb</groupId>
+ <artifactId>jaxb-runtime</artifactId>
+ <version>${jaxb-runtime.version}</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git
a/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/HiveRESTCatalogServerExtension.java
b/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/HiveRESTCatalogServerExtension.java
index bc39ab612ea..771e03f9c85 100644
---
a/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/HiveRESTCatalogServerExtension.java
+++
b/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/HiveRESTCatalogServerExtension.java
@@ -24,33 +24,63 @@
import org.apache.hadoop.hive.metastore.ServletSecurity.AuthType;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.iceberg.rest.extension.OAuth2AuthorizationServer;
import org.apache.iceberg.rest.extension.RESTCatalogServer;
import org.junit.rules.ExternalResource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
public class HiveRESTCatalogServerExtension extends ExternalResource {
private final Configuration conf;
+ private final OAuth2AuthorizationServer authorizationServer;
private final RESTCatalogServer restCatalogServer;
- private HiveRESTCatalogServerExtension(AuthType authType, Class<? extends
MetaStoreSchemaInfo> schemaInfoClass) {
+ static final String HMS_ID = "hive-metastore";
+ static final String HMS_SECRET = "hive-metastore-secret";
+
+ private static final Logger LOG =
LoggerFactory.getLogger(HiveRESTCatalogServerExtension.class);
+
+ private HiveRESTCatalogServerExtension(AuthType authType, Class<? extends
MetaStoreSchemaInfo> schemaInfoClass,
+ Map<String, String> configurations) {
this.conf = MetastoreConf.newMetastoreConf();
MetastoreConf.setVar(conf, ConfVars.CATALOG_SERVLET_AUTH, authType.name());
+ if (authType == AuthType.OAUTH2) {
+ authorizationServer = new OAuth2AuthorizationServer();
+ MetastoreConf.setVar(conf, ConfVars.CATALOG_SERVLET_AUTH, "oauth2");
+ MetastoreConf.setVar(conf,
ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_CLIENT_ID, HMS_ID);
+ MetastoreConf.setVar(conf,
ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_CLIENT_SECRET, HMS_SECRET);
+ MetastoreConf.setVar(conf,
ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_AUDIENCE, HMS_ID);
+ MetastoreConf.setVar(conf,
ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_PRINCIPAL_MAPPER_REGEX_FIELD, "email");
+ MetastoreConf.setVar(conf,
ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_PRINCIPAL_MAPPER_REGEX_PATTERN,
+ "(.*)@example.com");
+ } else {
+ authorizationServer = null;
+ }
+ configurations.forEach(conf::set);
restCatalogServer = new RESTCatalogServer();
if (schemaInfoClass != null) {
restCatalogServer.setSchemaInfoClass(schemaInfoClass);
}
}
- public Configuration getConf() {
- return conf;
- }
-
@Override
protected void before() throws Throwable {
+ if (authorizationServer != null) {
+ authorizationServer.start();
+ LOG.info("An authorization server {} started",
authorizationServer.getIssuer());
+ MetastoreConf.setVar(conf, ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_ISSUER,
authorizationServer.getIssuer());
+ }
restCatalogServer.start(conf);
}
@Override
protected void after() {
+ if (authorizationServer != null) {
+ authorizationServer.stop();
+ }
restCatalogServer.stop();
}
@@ -58,9 +88,18 @@ public String getRestEndpoint() {
return restCatalogServer.getRestEndpoint();
}
+ public String getOAuth2TokenEndpoint() {
+ return authorizationServer.getTokenEndpoint();
+ }
+
+ public String getOAuth2ClientCredential() {
+ return authorizationServer.getClientCredential();
+ }
+
public static class Builder {
private final AuthType authType;
private Class<? extends MetaStoreSchemaInfo> metaStoreSchemaClass;
+ private final Map<String, String> configurations = new HashMap<>();
private Builder(AuthType authType) {
this.authType = authType;
@@ -72,7 +111,7 @@ public Builder addMetaStoreSchemaClassName(Class<? extends
MetaStoreSchemaInfo>
}
public HiveRESTCatalogServerExtension build() {
- return new HiveRESTCatalogServerExtension(authType,
metaStoreSchemaClass);
+ return new HiveRESTCatalogServerExtension(authType,
metaStoreSchemaClass, configurations);
}
}
diff --git
a/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergRESTCatalogGravitinoLlapLocalCliDriver.java
b/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergRESTCatalogGravitinoLlapLocalCliDriver.java
index 70b9985ce0f..07a49d3a271 100644
---
a/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergRESTCatalogGravitinoLlapLocalCliDriver.java
+++
b/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergRESTCatalogGravitinoLlapLocalCliDriver.java
@@ -30,6 +30,7 @@
import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.hive.CatalogUtils;
import org.apache.iceberg.hive.client.HiveRESTCatalogClient;
+import org.apache.iceberg.rest.extension.OAuth2AuthorizationServer;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
@@ -41,6 +42,7 @@
import org.junit.runners.Parameterized.Parameters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.Network;
import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.containers.wait.strategy.WaitAllStrategy;
@@ -80,12 +82,16 @@ public class
TestIcebergRESTCatalogGravitinoLlapLocalCliDriver {
private static final DockerImageName GRAVITINO_IMAGE =
DockerImageName.parse("apache/gravitino-iceberg-rest:1.0.0");
+ private static final String OAUTH2_SERVER_ICEBERG_CLIENT_ID =
"iceberg-client";
+ private static final String OAUTH2_SERVER_ICEBERG_CLIENT_SECRET =
"iceberg-client-secret";
+
private final String name;
private final File qfile;
private GenericContainer<?> gravitinoContainer;
private Path warehouseDir;
private final ScheduledExecutorService fileSyncExecutor =
Executors.newSingleThreadScheduledExecutor();
+ private OAuth2AuthorizationServer oAuth2AuthorizationServer;
@Parameters(name = "{0}")
public static List<Object[]> getParameters() throws Exception {
@@ -105,9 +111,12 @@ public
TestIcebergRESTCatalogGravitinoLlapLocalCliDriver(String name, File qfile
@Before
public void setup() throws IOException {
+ Network dockerNetwork = Network.newNetwork();
+
+ startOAuth2AuthorizationServer(dockerNetwork);
createWarehouseDir();
prepareGravitinoConfig();
- startGravitinoContainer();
+ startGravitinoContainer(dockerNetwork);
fileSyncExecutor.scheduleAtFixedRate(this::syncWarehouseDir, 0, 5,
TimeUnit.SECONDS);
String host = gravitinoContainer.getHost();
@@ -123,6 +132,11 @@ public void setup() throws IOException {
MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CATALOG_DEFAULT,
CATALOG_NAME);
conf.set(restCatalogPrefix + "uri", restCatalogUri);
conf.set(restCatalogPrefix + "type",
CatalogUtil.ICEBERG_CATALOG_TYPE_REST);
+
+ // OAUTH2 Configs
+ conf.set(restCatalogPrefix + "rest.auth.type", "oauth2");
+ conf.set(restCatalogPrefix + "oauth2-server-uri",
oAuth2AuthorizationServer.getTokenEndpoint());
+ conf.set(restCatalogPrefix + "credential",
oAuth2AuthorizationServer.getClientCredential());
}
@After
@@ -130,6 +144,10 @@ public void teardown() throws IOException {
if (gravitinoContainer != null) {
gravitinoContainer.stop();
}
+
+ if (oAuth2AuthorizationServer != null) {
+ oAuth2AuthorizationServer.stop();
+ }
fileSyncExecutor.shutdownNow();
FileUtils.deleteDirectory(warehouseDir.toFile());
@@ -157,7 +175,7 @@ public void teardown() throws IOException {
* multiple test methods rather than being confined to a single block
scope.</p>
*/
@SuppressWarnings("resource")
- private void startGravitinoContainer() {
+ private void startGravitinoContainer(Network dockerNetwork) {
gravitinoContainer = new GenericContainer<>(GRAVITINO_IMAGE)
.withExposedPorts(GRAVITINO_HTTP_PORT)
// Update entrypoint to create the warehouse directory before starting
the server
@@ -175,6 +193,8 @@ private void startGravitinoContainer() {
),
GRAVITINO_H2_LIB
)
+ // Use the same Docker network as the OAuth2 server so they can
communicate
+ .withNetwork(dockerNetwork)
// Wait for the server to be fully started
.waitingFor(
new WaitAllStrategy()
@@ -254,6 +274,11 @@ private void syncWarehouseDir() {
}
}
}
+
+ private void startOAuth2AuthorizationServer(Network dockerNetwork) {
+ oAuth2AuthorizationServer = new OAuth2AuthorizationServer(dockerNetwork,
false);
+ oAuth2AuthorizationServer.start();
+ }
private void createWarehouseDir() {
try {
@@ -276,12 +301,26 @@ private void prepareGravitinoConfig() throws IOException {
String updatedContent = content
.replace("/WAREHOUSE_DIR", warehouseDir.toString())
+ .replace("OAUTH2_SERVER_URI", oAuth2AuthorizationServer.getIssuer())
+ .replace("OAUTH2_JWKS_URI", getJwksUri())
+ .replace("OAUTH2_CLIENT_ID", OAUTH2_SERVER_ICEBERG_CLIENT_ID)
+ .replace("OAUTH2_CLIENT_SECRET", OAUTH2_SERVER_ICEBERG_CLIENT_SECRET)
.replace("HTTP_PORT", String.valueOf(GRAVITINO_HTTP_PORT));
Path configFile = warehouseDir.resolve(GRAVITINO_CONF_FILE_TEMPLATE);
Files.writeString(configFile, updatedContent);
}
+ private String getJwksUri() {
+ String reachableHost =
oAuth2AuthorizationServer.getKeycloackContainerDockerInternalHostName();
+ int internalPort = 8080; // Keycloak container's internal port
+ return oAuth2AuthorizationServer.getIssuer()
+ .replace("localhost", reachableHost)
+ .replace("127.0.0.1", reachableHost)
+ // replace issuer's mapped port with keyclock container's internal port
+ .replaceFirst(":[0-9]+", ":" + internalPort);
+ }
+
@Test
public void testCliDriver() throws Exception {
CLI_ADAPTER.runTest(name, qfile);
diff --git
a/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergRESTCatalogHMSLlapLocalCliDriver.java
b/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergRESTCatalogHMSLlapLocalCliDriver.java
index 66a2985f4f9..2f5031601de 100644
---
a/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergRESTCatalogHMSLlapLocalCliDriver.java
+++
b/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergRESTCatalogHMSLlapLocalCliDriver.java
@@ -28,6 +28,7 @@
import org.apache.hive.ITestsSchemaInfo;
import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.hive.CatalogUtils;
+import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Rule;
@@ -36,21 +37,14 @@
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
import java.util.List;
-import java.util.stream.Stream;
@RunWith(Parameterized.class)
public class TestIcebergRESTCatalogHMSLlapLocalCliDriver {
- private static final Logger LOG = LoggerFactory.getLogger(
- TestIcebergRESTCatalogHMSLlapLocalCliDriver.class);
private static final String CATALOG_NAME = "ice01";
private static final CliAdapter CLI_ADAPTER =
new
CliConfigs.TestIcebergRESTCatalogHMSLlapLocalCliDriver().getCliAdapter();
@@ -60,7 +54,7 @@ public class TestIcebergRESTCatalogHMSLlapLocalCliDriver {
@ClassRule
public static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
- HiveRESTCatalogServerExtension.builder(ServletSecurity.AuthType.NONE)
+ HiveRESTCatalogServerExtension.builder(ServletSecurity.AuthType.OAUTH2)
.addMetaStoreSchemaClassName(ITestsSchemaInfo.class)
.build();
@@ -90,26 +84,16 @@ public void setupHiveConfig() {
MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CATALOG_DEFAULT,
CATALOG_NAME);
conf.set(restCatalogPrefix + "uri",
REST_CATALOG_EXTENSION.getRestEndpoint());
conf.set(restCatalogPrefix + "type",
CatalogUtil.ICEBERG_CATALOG_TYPE_REST);
- }
- @Before
- public void cleanUpRestCatalogServerTmpDir() throws IOException {
- try (Stream<Path> children =
Files.list(REST_CATALOG_EXTENSION.getRestCatalogServer().getWarehouseDir())) {
- children
- .filter(path -> !path.getFileName().toString().equals("derby.log"))
- .filter(path ->
!path.getFileName().toString().equals("metastore_db"))
- .forEach(path -> {
- try {
- if (Files.isDirectory(path)) {
- FileUtils.deleteDirectory(path.toFile());
- } else {
- Files.delete(path);
- }
- } catch (IOException e) {
- LOG.error("Failed to delete path: {}", path, e);
- }
- });
- }
+ // auth configs
+ conf.set(restCatalogPrefix + "rest.auth.type", "oauth2");
+ conf.set(restCatalogPrefix + "oauth2-server-uri",
REST_CATALOG_EXTENSION.getOAuth2TokenEndpoint());
+ conf.set(restCatalogPrefix + "credential",
REST_CATALOG_EXTENSION.getOAuth2ClientCredential());
+ }
+
+ @After
+ public void tearDown() throws IOException {
+
FileUtils.deleteDirectory(REST_CATALOG_EXTENSION.getRestCatalogServer().getWarehouseDir().toFile());
}
@Test
diff --git
a/itests/qtest-iceberg/src/test/resources/gravitino-h2-test-template.conf
b/itests/qtest-iceberg/src/test/resources/gravitino-h2-test-template.conf
index 12009e7fa8f..7d88014eea7 100644
--- a/itests/qtest-iceberg/src/test/resources/gravitino-h2-test-template.conf
+++ b/itests/qtest-iceberg/src/test/resources/gravitino-h2-test-template.conf
@@ -11,4 +11,20 @@ gravitino.iceberg-rest.jdbc-password = ""
gravitino.iceberg-rest.jdbc-initialize = true
# --- Warehouse Location (where data files are stored) ---
-gravitino.iceberg-rest.warehouse = file:///WAREHOUSE_DIR/iceberg_warehouse
\ No newline at end of file
+gravitino.iceberg-rest.warehouse = file:///WAREHOUSE_DIR/iceberg_warehouse
+
+# --- OAuth2 Authentication ---
+gravitino.authenticators = oauth
+
+gravitino.authenticator.oauth.serverUri = OAUTH2_SERVER_URI
+gravitino.authenticator.oauth.tokenPath = /protocol/openid-connect/token
+gravitino.authenticator.oauth.clientId = OAUTH2_CLIENT_ID
+gravitino.authenticator.oauth.scope = openid catalog
+gravitino.authenticator.oauth.clientSecret = OAUTH2_CLIENT_SECRET
+
+gravitino.authenticator.oauth.tokenValidatorClass =
org.apache.gravitino.server.authentication.JwksTokenValidator
+gravitino.authenticator.oauth.jwksUri =
OAUTH2_JWKS_URI/protocol/openid-connect/certs
+gravitino.authenticator.oauth.provider = default
+gravitino.authenticator.oauth.principalFields = sub
+gravitino.authenticator.oauth.allowSkewSecs = 60
+gravitino.authenticator.oauth.serviceAudience = hive-metastore
diff --git a/pom.xml b/pom.xml
index 28bc59c5c77..b2ece4d6c4b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -240,6 +240,8 @@
<spring.version>5.3.39</spring.version>
<spring.ldap.version>2.4.4</spring.ldap.version>
<project.build.outputTimestamp>2025-01-01T00:00:00Z</project.build.outputTimestamp>
+ <keycloak.version>26.0.6</keycloak.version>
+ <nimbus-oauth.version>11.28</nimbus-oauth.version>
</properties>
<repositories>
<!-- This needs to be removed before checking in-->
diff --git
a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/OAuth2AuthorizationServer.java
b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/OAuth2AuthorizationServer.java
index 4f7731d2569..4f339e035cc 100644
---
a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/OAuth2AuthorizationServer.java
+++
b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/OAuth2AuthorizationServer.java
@@ -32,6 +32,7 @@
import org.keycloak.representations.idm.ProtocolMapperRepresentation;
import org.keycloak.representations.idm.RealmRepresentation;
import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
import org.testcontainers.utility.DockerImageName;
public class OAuth2AuthorizationServer {
@@ -46,6 +47,18 @@ public class OAuth2AuthorizationServer {
private String issuer;
private String tokenEndpoint;
private String accessToken;
+ private final boolean accessTokenHeaderTypeRfc9068;
+ private final Network dockerNetwork;
+
+ public OAuth2AuthorizationServer(Network dockerNetwork, boolean
accessTokenHeaderTypeRfc9068) {
+ this.dockerNetwork = dockerNetwork;
+ this.accessTokenHeaderTypeRfc9068 = accessTokenHeaderTypeRfc9068;
+ }
+
+ public OAuth2AuthorizationServer() {
+ dockerNetwork = Network.newNetwork();
+ accessTokenHeaderTypeRfc9068 = true;
+ }
private static RealmResource createRealm(Keycloak keycloak) {
var realm = new RealmRepresentation();
@@ -100,7 +113,7 @@ private static ProtocolMapperRepresentation
createEmailClaim() {
return mapper;
}
- private static void createClient(RealmResource realm, List<String> scopes,
+ private void createClient(RealmResource realm, List<String> scopes,
List<ProtocolMapperRepresentation> protocolMappers) {
var client = new ClientRepresentation();
client.setClientId(ICEBERG_CLIENT_ID);
@@ -110,7 +123,8 @@ private static void createClient(RealmResource realm,
List<String> scopes,
client.setPublicClient(false);
client.setServiceAccountsEnabled(true);
client.setOptionalClientScopes(scopes);
-
client.setAttributes(Collections.singletonMap("access.token.header.type.rfc9068",
"true"));
+
client.setAttributes(Collections.singletonMap("access.token.header.type.rfc9068",
+ Boolean.valueOf(accessTokenHeaderTypeRfc9068).toString()));
client.setProtocolMappers(protocolMappers);
realm.clients().create(client).close();
}
@@ -128,12 +142,13 @@ private static String getAccessToken(String url,
List<String> scopes) {
}
}
- void start() {
+ public void start() {
container = new
GenericContainer<>(DockerImageName.parse("quay.io/keycloak/keycloak:26.3.4"))
.withEnv("KEYCLOAK_ADMIN", "admin")
.withEnv("KEYCLOAK_ADMIN_PASSWORD", "admin")
.withCommand("start-dev")
.withExposedPorts(8080)
+ .withNetwork(dockerNetwork)
.withStartupTimeout(Duration.ofMinutes(5));
container.start();
@@ -152,26 +167,30 @@ void start() {
accessToken = getAccessToken(base, List.of("catalog"));
}
- void stop() {
+ public void stop() {
if (container != null) {
container.stop();
keycloak.close();
}
}
- String getIssuer() {
+ public String getIssuer() {
return issuer;
}
- String getTokenEndpoint() {
+ public String getTokenEndpoint() {
return tokenEndpoint;
}
- String getClientCredential() {
+ public String getClientCredential() {
return "%s:%s".formatted(ICEBERG_CLIENT_ID, ICEBERG_CLIENT_SECRET);
}
- String getAccessToken() {
+ public String getAccessToken() {
return accessToken;
}
+
+ public String getKeycloackContainerDockerInternalHostName() {
+ return container.getNetworkAliases().get(0);
+ }
}
