This is an automated email from the ASF dual-hosted git repository.
ajantha pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris-tools.git
The following commit(s) were added to refs/heads/main by this push:
new 7b7f280 Add polaris catalog migrator tests (#32)
7b7f280 is described below
commit 7b7f280f5387d43c2c17586ff8b831a65402c478
Author: Ajantha Bhat <[email protected]>
AuthorDate: Tue Oct 7 07:10:21 2025 +0530
Add polaris catalog migrator tests (#32)
Added Integration tests with Polaris docker image to migrate tables from
Nessie, Hadoop, Hive to polaris.
---
.../catalog/migrator/api/test/AbstractTest.java | 16 +-
iceberg-catalog-migrator/cli/build.gradle.kts | 2 +
.../migrator/cli/AbstractCLIMigrationTest.java | 16 +-
.../migrator/cli/HadoopCLIMigrationTest.java | 6 +
.../cli/ITHadoopToNessieCLIMigrationTest.java | 6 +
...java => ITHadoopToPolarisCLIMigrationTest.java} | 36 +++-
.../cli/ITHiveToPolarisCLIMigrationTest.java | 72 ++++++++
.../cli/ITNessieToPolarisCLIMigrationTest.java | 62 +++++++
.../catalog/migrator/cli/PolarisContainer.java | 202 +++++++++++++++++++++
iceberg-catalog-migrator/gradle/libs.versions.toml | 1 +
10 files changed, 401 insertions(+), 18 deletions(-)
diff --git
a/iceberg-catalog-migrator/api-test/src/main/java/org/apache/polaris/iceberg/catalog/migrator/api/test/AbstractTest.java
b/iceberg-catalog-migrator/api-test/src/main/java/org/apache/polaris/iceberg/catalog/migrator/api/test/AbstractTest.java
index 9bc7d3d..331f58e 100644
---
a/iceberg-catalog-migrator/api-test/src/main/java/org/apache/polaris/iceberg/catalog/migrator/api/test/AbstractTest.java
+++
b/iceberg-catalog-migrator/api-test/src/main/java/org/apache/polaris/iceberg/catalog/migrator/api/test/AbstractTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.polaris.iceberg.catalog.migrator.api.test;
+import java.io.File;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.HashMap;
@@ -55,8 +56,8 @@ public abstract class AbstractTest {
protected static final Namespace NS_A_B_C_D = Namespace.of("a", "b", "c",
"d");
protected static final Namespace NS_A_B_C_D_E = Namespace.of("a", "b", "c",
"d", "e");
- private static String sourceCatalogWarehouse;
- private static String targetCatalogWarehouse;
+ protected static String sourceCatalogWarehouse;
+ protected static String targetCatalogWarehouse;
protected static Catalog sourceCatalog;
protected static Catalog targetCatalog;
@@ -73,7 +74,9 @@ public abstract class AbstractTest {
protected static void initLogDir() {
System.setProperty("catalog.migration.log.dir",
logDir.toAbsolutePath().toString());
sourceCatalogWarehouse =
tempDir.resolve("sourceCatalogWarehouse").toAbsolutePath().toString();
+ ensureDirectoryExists(sourceCatalogWarehouse);
targetCatalogWarehouse =
tempDir.resolve("targetCatalogWarehouse").toAbsolutePath().toString();
+ ensureDirectoryExists(targetCatalogWarehouse);
}
@AfterAll
@@ -156,4 +159,13 @@ public abstract class AbstractTest {
properties.putAll(dynamicProperties);
return properties;
}
+
+ private static void ensureDirectoryExists(String path) {
+ File dir = new File(path);
+ if (!dir.exists()) {
+ if (!dir.mkdirs()) {
+ throw new RuntimeException("Unable to create directory: " + path);
+ }
+ }
+ }
}
diff --git a/iceberg-catalog-migrator/cli/build.gradle.kts
b/iceberg-catalog-migrator/cli/build.gradle.kts
index b753412..08fae96 100644
--- a/iceberg-catalog-migrator/cli/build.gradle.kts
+++ b/iceberg-catalog-migrator/cli/build.gradle.kts
@@ -128,6 +128,8 @@ dependencies {
}
testImplementation("org.apache.hadoop:hadoop-mapreduce-client-core:${libs.versions.hadoop.get()}")
+
testImplementation("org.testcontainers:testcontainers:${libs.versions.testcontainers.get()}")
+
nessieQuarkusServer(
"org.projectnessie.nessie:nessie-quarkus:${libs.versions.nessie.get()}:runner"
)
diff --git
a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/AbstractCLIMigrationTest.java
b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/AbstractCLIMigrationTest.java
index 57f85ac..c9c39f3 100644
---
a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/AbstractCLIMigrationTest.java
+++
b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/AbstractCLIMigrationTest.java
@@ -32,13 +32,11 @@ import java.util.Map;
import nl.altindag.log.LogCaptor;
import nl.altindag.log.model.LogEvent;
import org.apache.iceberg.catalog.Catalog;
-import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.exceptions.NoSuchTableException;
import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrator;
import org.apache.polaris.iceberg.catalog.migrator.api.test.AbstractTest;
import org.assertj.core.api.Assertions;
-import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.io.TempDir;
@@ -83,6 +81,9 @@ public abstract class AbstractCLIMigrationTest extends
AbstractTest {
case HIVE:
properties = hiveCatalogProperties(isSourceCatalog, additionalProp);
break;
+ case REST:
+ properties = additionalProp;
+ break;
default:
throw new UnsupportedOperationException(
String.format("Unsupported for catalog type: %s", catalogType));
@@ -106,11 +107,6 @@ public abstract class AbstractCLIMigrationTest extends
AbstractTest {
}
}
- @AfterAll
- protected static void tearDown() throws Exception {
- dropNamespaces();
- }
-
@BeforeEach
protected void beforeEach() {
createTables();
@@ -118,12 +114,6 @@ public abstract class AbstractCLIMigrationTest extends
AbstractTest {
@AfterEach
protected void afterEach() {
- // manually refreshing catalog due to missing refresh in Nessie catalog
- // https://github.com/apache/iceberg/pull/6789
- // create table will call refresh internally.
- sourceCatalog.createTable(TableIdentifier.of(BAR, "tblx"),
schema).refresh();
- targetCatalog.createTable(TableIdentifier.of(BAR, "tblx"),
schema).refresh();
-
dropTables();
}
diff --git
a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/HadoopCLIMigrationTest.java
b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/HadoopCLIMigrationTest.java
index 0e6d6c4..1c6de27 100644
---
a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/HadoopCLIMigrationTest.java
+++
b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/HadoopCLIMigrationTest.java
@@ -20,6 +20,7 @@ package org.apache.polaris.iceberg.catalog.migrator.cli;
import java.util.Collections;
import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
public class HadoopCLIMigrationTest extends AbstractCLIMigrationTest {
@@ -29,4 +30,9 @@ public class HadoopCLIMigrationTest extends
AbstractCLIMigrationTest {
initializeSourceCatalog(CatalogMigrationUtil.CatalogType.HADOOP,
Collections.emptyMap());
initializeTargetCatalog(CatalogMigrationUtil.CatalogType.HADOOP,
Collections.emptyMap());
}
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
}
diff --git
a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToNessieCLIMigrationTest.java
b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToNessieCLIMigrationTest.java
index 938df44..6973ba1 100644
---
a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToNessieCLIMigrationTest.java
+++
b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToNessieCLIMigrationTest.java
@@ -27,6 +27,7 @@ import org.apache.iceberg.catalog.SupportsNamespaces;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@ -38,6 +39,11 @@ public class ITHadoopToNessieCLIMigrationTest extends
AbstractCLIMigrationTest {
initializeTargetCatalog(CatalogMigrationUtil.CatalogType.NESSIE,
Collections.emptyMap());
}
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+
@Test
public void testRegisterLargeNumberOfTablesWithNestedNamespaces() throws
Exception {
List<Namespace> namespaceList =
diff --git
a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/HadoopCLIMigrationTest.java
b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToPolarisCLIMigrationTest.java
similarity index 51%
copy from
iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/HadoopCLIMigrationTest.java
copy to
iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToPolarisCLIMigrationTest.java
index 0e6d6c4..97303a0 100644
---
a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/HadoopCLIMigrationTest.java
+++
b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToPolarisCLIMigrationTest.java
@@ -18,15 +18,45 @@
*/
package org.apache.polaris.iceberg.catalog.migrator.cli;
+import static org.assertj.core.api.Assertions.assertThat;
+
import java.util.Collections;
+import java.util.Map;
import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
-public class HadoopCLIMigrationTest extends AbstractCLIMigrationTest {
+public class ITHadoopToPolarisCLIMigrationTest extends
AbstractCLIMigrationTest {
+
+ private static PolarisContainer polarisContainer;
@BeforeAll
- protected static void setup() {
+ protected static void setup() throws Exception {
+ polarisContainer = new PolarisContainer(sourceCatalogWarehouse);
+ polarisContainer.start();
+
+ assertThat(polarisContainer.httpGet("/api/management/v1/catalogs"))
+ .contains(PolarisContainer.CATALOG_NAME);
+
initializeSourceCatalog(CatalogMigrationUtil.CatalogType.HADOOP,
Collections.emptyMap());
- initializeTargetCatalog(CatalogMigrationUtil.CatalogType.HADOOP,
Collections.emptyMap());
+
+ initializeTargetCatalog(
+ CatalogMigrationUtil.CatalogType.REST,
+ Map.of(
+ "uri",
+ polarisContainer.getIcebergApiEndpoint(),
+ "warehouse",
+ PolarisContainer.CATALOG_NAME,
+ "token",
+ polarisContainer.getAccessToken(
+ polarisContainer.getClientId(),
polarisContainer.getClientSecret())));
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ if (polarisContainer != null) {
+ polarisContainer.stop();
+ }
}
}
diff --git
a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHiveToPolarisCLIMigrationTest.java
b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHiveToPolarisCLIMigrationTest.java
new file mode 100644
index 0000000..d1e8621
--- /dev/null
+++
b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHiveToPolarisCLIMigrationTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.Collections;
+import java.util.Map;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+@Disabled("Because of https://github.com/apache/polaris/issues/2756";)
+public class ITHiveToPolarisCLIMigrationTest extends AbstractCLIMigrationTest {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ private static PolarisContainer polarisContainer;
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ polarisContainer = new PolarisContainer(sourceCatalogWarehouse);
+ polarisContainer.start();
+ assertThat(polarisContainer.httpGet("/api/management/v1/catalogs"))
+ .contains(PolarisContainer.CATALOG_NAME);
+
+ initializeSourceCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri",
HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+
+ initializeTargetCatalog(
+ CatalogMigrationUtil.CatalogType.REST,
+ Map.of(
+ "uri",
+ polarisContainer.getIcebergApiEndpoint(),
+ "warehouse",
+ PolarisContainer.CATALOG_NAME,
+ "token",
+ polarisContainer.getAccessToken(
+ polarisContainer.getClientId(),
polarisContainer.getClientSecret())));
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ if (polarisContainer != null) {
+ polarisContainer.stop();
+ }
+ }
+}
diff --git
a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITNessieToPolarisCLIMigrationTest.java
b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITNessieToPolarisCLIMigrationTest.java
new file mode 100644
index 0000000..3738baa
--- /dev/null
+++
b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITNessieToPolarisCLIMigrationTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.Collections;
+import java.util.Map;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+
+public class ITNessieToPolarisCLIMigrationTest extends
AbstractCLIMigrationTest {
+
+ private static PolarisContainer polarisContainer;
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ polarisContainer = new PolarisContainer(sourceCatalogWarehouse);
+ polarisContainer.start();
+ assertThat(polarisContainer.httpGet("/api/management/v1/catalogs"))
+ .contains(PolarisContainer.CATALOG_NAME);
+
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.NESSIE,
Collections.emptyMap());
+
+ initializeTargetCatalog(
+ CatalogMigrationUtil.CatalogType.REST,
+ Map.of(
+ "uri",
+ polarisContainer.getIcebergApiEndpoint(),
+ "warehouse",
+ PolarisContainer.CATALOG_NAME,
+ "token",
+ polarisContainer.getAccessToken(
+ polarisContainer.getClientId(),
polarisContainer.getClientSecret())));
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+
+ if (polarisContainer != null) {
+ polarisContainer.stop();
+ }
+ }
+}
diff --git
a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/PolarisContainer.java
b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/PolarisContainer.java
new file mode 100644
index 0000000..699e82d
--- /dev/null
+++
b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/PolarisContainer.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import static org.junit.Assert.assertEquals;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Preconditions;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.testcontainers.containers.BindMode;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.utility.DockerImageName;
+
+public class PolarisContainer extends GenericContainer<PolarisContainer> {
+
+ private static final DockerImageName IMAGE =
+ DockerImageName.parse("apache/polaris:1.1.0-incubating");
+ private static final int POLARIS_PORT = 8181;
+
+ public static final String CATALOG_NAME = "test";
+ public static final String NAMESPACE_NAME = "testNamespace";
+ private static final HttpClient HTTP_CLIENT = HttpClient.newHttpClient();
+
+ private static final String CLIENT_ID = "root";
+ private static final String CLIENT_SECRET = "s3cr3t";
+
+ private final Map<String, String> jvmOptions = new HashMap<>();
+
+ private final String hostDirectory;
+
+ public PolarisContainer(String hostDirectory) {
+ super(IMAGE);
+
+ Preconditions.checkArgument(hostDirectory != null, "host directory is
null");
+ this.hostDirectory = hostDirectory;
+ this.withFileSystemBind(hostDirectory, hostDirectory, BindMode.READ_WRITE);
+ Wait.forHttp("/").forStatusCode(200);
+ this.withExposedPorts(POLARIS_PORT).withEnv("JAVA_TOOL_OPTIONS",
"-Duser.dir=" + hostDirectory);
+ }
+
+ @Override
+ public void start() {
+ // setup to allow file storage
+ jvmOptions.put("polaris.readiness.ignore-severe-issues", "true");
+ jvmOptions.put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
"[\"FILE\"]");
+ jvmOptions.put("polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
"true");
+ String jvmOptionsString =
+ jvmOptions.entrySet().stream()
+ .map(e -> "-D" + e.getKey() + "=" + e.getValue())
+ .collect(Collectors.joining(" "));
+ this.addEnv("JAVA_OPTS_APPEND", jvmOptionsString);
+ this.addEnv(
+ "POLARIS_BOOTSTRAP_CREDENTIALS", String.format("POLARIS,%s,%s",
CLIENT_ID, CLIENT_SECRET));
+ super.start();
+ createCatalog(CATALOG_NAME, hostDirectory);
+ createNamespace(NAMESPACE_NAME);
+ }
+
+ @Override
+ public void setDockerImageName(String dockerImageName) {
+ throw new UnsupportedOperationException("Docker image name can not be
changed");
+ }
+
+ public String getApiEndpoint(String prefix) {
+ return "http://localhost:"; + getMappedPort(POLARIS_PORT) + prefix;
+ }
+
+ public String getIcebergApiEndpoint() {
+ return getApiEndpoint("/api/catalog");
+ }
+
+ public String getClientId() {
+ return CLIENT_ID;
+ }
+
+ public String getClientSecret() {
+ return CLIENT_SECRET;
+ }
+
+ public String httpGet(String apiUrl) throws Exception {
+ String accessToken = getAccessToken(CLIENT_ID, CLIENT_SECRET);
+
+ HttpRequest request =
+ HttpRequest.newBuilder()
+ .uri(URI.create(getApiEndpoint(apiUrl)))
+ .header("Content-Type", "application/json")
+ .header("Authorization", "Bearer " + accessToken)
+ .GET()
+ .build();
+ HttpResponse<String> response = HTTP_CLIENT.send(request,
HttpResponse.BodyHandlers.ofString());
+ assertEquals(response.body(), 200, response.statusCode());
+ return response.body();
+ }
+
+ public String getAccessToken(String clientId, String clientSecret) {
+ try {
+ String body =
+ String.format(
+
"grant_type=client_credentials&client_id=%s&client_secret=%s&scope=PRINCIPAL_ROLE:ALL",
+ clientId, clientSecret);
+ HttpRequest request =
+ HttpRequest.newBuilder()
+ .uri(URI.create(getApiEndpoint("/api/catalog/v1/oauth/tokens")))
+ .header("Content-Type", "application/x-www-form-urlencoded")
+ .header("Accept", "application/json")
+ .POST(HttpRequest.BodyPublishers.ofString(body))
+ .build();
+
+ HttpResponse<String> response =
+ HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofString());
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode root = mapper.readTree(response.body());
+ return root.path("access_token").asText();
+ } catch (Exception e) {
+ throw new AssertionError("Error fetching access token");
+ }
+ }
+
+ public void createCatalog(String catalogName, String hostDirectory) {
+ try {
+ String body =
+ String.format(
+ "{\n"
+ + " \"catalog\": {\n"
+ + " \"name\": \"%s\",\n"
+ + " \"type\": \"INTERNAL\",\n"
+ + " \"readOnly\": false,\n"
+ + " \"properties\": {\n"
+ + " \"default-base-location\": \"file:%s/\"\n"
+ + " },\n"
+ + " \"storageConfigInfo\": {\n"
+ + " \"storageType\": \"FILE\",\n"
+ + " \"allowedLocations\": [\n"
+ + " \"file:%s\"\n"
+ + " ]\n"
+ + " }\n"
+ + " }\n"
+ + " }",
+ catalogName, hostDirectory, hostDirectory);
+ HttpRequest request =
+ HttpRequest.newBuilder()
+ .uri(URI.create(getApiEndpoint("/api/management/v1/catalogs")))
+ .header("Content-Type", "application/json")
+ .header("Authorization", "Bearer " + getAccessToken(CLIENT_ID,
CLIENT_SECRET))
+ .header("Accept", "application/json")
+ .POST(HttpRequest.BodyPublishers.ofString(body))
+ .build();
+
+ HttpResponse<String> response =
+ HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofString());
+ assertEquals(response.body(), 201, response.statusCode());
+ } catch (Exception e) {
+ throw new AssertionError("Error creating catalog");
+ }
+ }
+
+ public void createNamespace(String namespaceName) {
+ try {
+ String body = String.format("{ \"namespace\": [\"%s\"]}", namespaceName);
+ HttpRequest request =
+ HttpRequest.newBuilder()
+ .uri(
+ URI.create(
+
getApiEndpoint(String.format("/api/catalog/v1/%s/namespaces", CATALOG_NAME))))
+ .header("Content-Type", "application/json")
+ .header("Authorization", "Bearer " + getAccessToken(CLIENT_ID,
CLIENT_SECRET))
+ .header("Accept", "application/json")
+ .POST(HttpRequest.BodyPublishers.ofString(body))
+ .build();
+
+ HttpResponse<String> response =
+ HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofString());
+ assertEquals(response.body(), 200, response.statusCode());
+ } catch (Exception e) {
+ throw new AssertionError("Error creating namespace");
+ }
+ }
+}
diff --git a/iceberg-catalog-migrator/gradle/libs.versions.toml
b/iceberg-catalog-migrator/gradle/libs.versions.toml
index cb40b53..adf3015 100644
--- a/iceberg-catalog-migrator/gradle/libs.versions.toml
+++ b/iceberg-catalog-migrator/gradle/libs.versions.toml
@@ -39,6 +39,7 @@ nessieRunner = "0.32.2"
picocli = "4.7.6"
shadowPlugin = "8.1.1"
slf4j = "1.7.36"
+testcontainers = "1.21.3"
[libraries]
assertj = { module = "org.assertj:assertj-core", version.ref = "assertj" }
