This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new d29f392e09 Nessie: Support ApiV2 for Nessie client (#6712)
d29f392e09 is described below
commit d29f392e09733b4d75c24f49bbd036d536a6c3e7
Author: Ajantha Bhat <[email protected]>
AuthorDate: Thu Jun 15 16:58:23 2023 +0530
Nessie: Support ApiV2 for Nessie client (#6712)
---
.../org/apache/iceberg/nessie/NessieCatalog.java | 24 +++++++++++++++++++---
.../java/org/apache/iceberg/nessie/NessieUtil.java | 2 ++
.../org/apache/iceberg/nessie/BaseTestIceberg.java | 7 +++++--
.../iceberg/nessie/TestCustomNessieClient.java | 8 ++++++--
.../apache/iceberg/nessie/TestNessieCatalog.java | 14 +++++++------
.../iceberg/nessie/TestNessieIcebergClient.java | 13 ++++++++++++
6 files changed, 55 insertions(+), 13 deletions(-)
diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java
b/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java
index 0c6f2d4c59..0c91cfb1db 100644
--- a/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java
+++ b/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java
@@ -44,6 +44,7 @@ import
org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.projectnessie.client.NessieClientBuilder;
import org.projectnessie.client.NessieConfigConstants;
import org.projectnessie.client.api.NessieApiV1;
+import org.projectnessie.client.api.NessieApiV2;
import org.projectnessie.client.http.HttpClientBuilder;
import org.projectnessie.model.ContentKey;
import org.projectnessie.model.TableReference;
@@ -88,11 +89,28 @@ public class NessieCatalog extends BaseMetastoreCatalog
options.get(removePrefix.apply(NessieConfigConstants.CONF_NESSIE_REF));
String requestedHash =
options.get(removePrefix.apply(NessieConfigConstants.CONF_NESSIE_REF_HASH));
- NessieApiV1 api =
+
+ NessieClientBuilder<?> nessieClientBuilder =
createNessieClientBuilder(
options.get(NessieConfigConstants.CONF_NESSIE_CLIENT_BUILDER_IMPL))
- .fromConfig(x -> options.get(removePrefix.apply(x)))
- .build(NessieApiV1.class);
+ .fromConfig(x -> options.get(removePrefix.apply(x)));
+ // default version is set to v1.
+ final String apiVersion =
+
options.getOrDefault(removePrefix.apply(NessieUtil.CLIENT_API_VERSION), "1");
+ NessieApiV1 api;
+ switch (apiVersion) {
+ case "1":
+ api = nessieClientBuilder.build(NessieApiV1.class);
+ break;
+ case "2":
+ api = nessieClientBuilder.build(NessieApiV2.class);
+ break;
+ default:
+ throw new IllegalArgumentException(
+ String.format(
+ "Unsupported %s: %s. Can only be 1 or 2",
+ removePrefix.apply(NessieUtil.CLIENT_API_VERSION),
apiVersion));
+ }
initialize(
name,
diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java
b/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java
index 0e57d8ccc3..0420597945 100644
--- a/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java
+++ b/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java
@@ -37,6 +37,8 @@ public final class NessieUtil {
public static final String NESSIE_CONFIG_PREFIX = "nessie.";
static final String APPLICATION_TYPE = "application-type";
+ public static final String CLIENT_API_VERSION = "nessie.client-api-version";
+
private NessieUtil() {}
static TableIdentifier removeCatalogName(TableIdentifier to, String name) {
diff --git
a/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java
b/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java
index ae4947a98c..5a9bbc2738 100644
--- a/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java
+++ b/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java
@@ -73,7 +73,7 @@ import org.slf4j.LoggerFactory;
@ExtendWith(DatabaseAdapterExtension.class)
@NessieDbAdapterName(InmemoryDatabaseAdapterFactory.NAME)
@NessieExternalDatabase(InmemoryTestConnectionProviderSource.class)
-@NessieApiVersions(versions = NessieApiVersion.V1)
+@NessieApiVersions // test all versions
public abstract class BaseTestIceberg {
@NessieDbAdapter static DatabaseAdapter databaseAdapter;
@@ -88,6 +88,7 @@ public abstract class BaseTestIceberg {
protected NessieCatalog catalog;
protected NessieApiV1 api;
+ protected String apiVersion;
protected Configuration hadoopConfig;
protected final String branch;
private String initialHashOfDefaultBranch;
@@ -122,6 +123,7 @@ public abstract class BaseTestIceberg {
throws IOException {
this.uri = nessieUri.toASCIIString();
this.api = clientFactory.make();
+ this.apiVersion = clientFactory.apiVersion() == NessieApiVersion.V2 ? "2"
: "1";
Branch defaultBranch = api.getDefaultBranch();
initialHashOfDefaultBranch = defaultBranch.getHash();
@@ -145,7 +147,8 @@ public abstract class BaseTestIceberg {
.put("ref", ref)
.put(CatalogProperties.URI, uri)
.put("auth-type", "NONE")
- .put(CatalogProperties.WAREHOUSE_LOCATION,
temp.toUri().toString());
+ .put(CatalogProperties.WAREHOUSE_LOCATION, temp.toUri().toString())
+ .put("client-api-version", apiVersion);
if (null != hash) {
options.put("ref.hash", hash);
}
diff --git
a/nessie/src/test/java/org/apache/iceberg/nessie/TestCustomNessieClient.java
b/nessie/src/test/java/org/apache/iceberg/nessie/TestCustomNessieClient.java
index 2a3c0c85e0..4034135807 100644
--- a/nessie/src/test/java/org/apache/iceberg/nessie/TestCustomNessieClient.java
+++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestCustomNessieClient.java
@@ -48,7 +48,9 @@ public class TestCustomNessieClient extends BaseTestIceberg {
CatalogProperties.WAREHOUSE_LOCATION,
temp.toUri().toString(),
CatalogProperties.URI,
- uri));
+ uri,
+ "client-api-version",
+ apiVersion));
}
@Test
@@ -62,7 +64,9 @@ public class TestCustomNessieClient extends BaseTestIceberg {
CatalogProperties.URI,
uri,
NessieConfigConstants.CONF_NESSIE_CLIENT_BUILDER_IMPL,
- HttpClientBuilder.class.getName()));
+ HttpClientBuilder.class.getName(),
+ "client-api-version",
+ apiVersion));
}
@Test
diff --git
a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java
b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java
index f8ed3987c1..8ff052712b 100644
--- a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java
+++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java
@@ -54,7 +54,7 @@ import
org.projectnessie.versioned.persist.tests.extension.NessieExternalDatabas
@ExtendWith(DatabaseAdapterExtension.class)
@NessieDbAdapterName(InmemoryDatabaseAdapterFactory.NAME)
@NessieExternalDatabase(InmemoryTestConnectionProviderSource.class)
-@NessieApiVersions(versions = NessieApiVersion.V1)
+@NessieApiVersions // test all versions
public class TestNessieCatalog extends CatalogTests<NessieCatalog> {
@NessieDbAdapter static DatabaseAdapter databaseAdapter;
@@ -67,6 +67,7 @@ public class TestNessieCatalog extends
CatalogTests<NessieCatalog> {
private NessieCatalog catalog;
private NessieApiV1 api;
+ private NessieApiVersion apiVersion;
private Configuration hadoopConfig;
private String initialHashOfDefaultBranch;
private String uri;
@@ -75,6 +76,7 @@ public class TestNessieCatalog extends
CatalogTests<NessieCatalog> {
public void setUp(NessieClientFactory clientFactory, @NessieClientUri URI
nessieUri)
throws NessieNotFoundException {
api = clientFactory.make();
+ apiVersion = clientFactory.apiVersion();
initialHashOfDefaultBranch = api.getDefaultBranch().getHash();
uri = nessieUri.toASCIIString();
hadoopConfig = new Configuration();
@@ -115,17 +117,17 @@ public class TestNessieCatalog extends
CatalogTests<NessieCatalog> {
private NessieCatalog initNessieCatalog(String ref) {
NessieCatalog newCatalog = new NessieCatalog();
newCatalog.setConf(hadoopConfig);
- newCatalog.initialize(
- "nessie",
+ ImmutableMap<String, String> options =
ImmutableMap.of(
"ref",
ref,
CatalogProperties.URI,
uri,
- "auth-type",
- "NONE",
CatalogProperties.WAREHOUSE_LOCATION,
- temp.toUri().toString()));
+ temp.toUri().toString(),
+ "client-api-version",
+ apiVersion == NessieApiVersion.V2 ? "2" : "1");
+ newCatalog.initialize("nessie", options);
return newCatalog;
}
diff --git
a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieIcebergClient.java
b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieIcebergClient.java
index d6643be91e..12fa807dcf 100644
---
a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieIcebergClient.java
+++
b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieIcebergClient.java
@@ -18,6 +18,7 @@
*/
package org.apache.iceberg.nessie;
+import java.io.IOException;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.assertj.core.api.Assertions;
@@ -89,4 +90,16 @@ public class TestNessieIcebergClient extends BaseTestIceberg
{
.isEqualTo(ref);
Assertions.assertThat(client.withReference(branch,
null)).isNotEqualTo(client);
}
+
+ @Test
+ public void testInvalidClientApiVersion() throws IOException {
+ try (NessieCatalog newCatalog = new NessieCatalog()) {
+ newCatalog.setConf(hadoopConfig);
+ ImmutableMap.Builder<String, String> options =
+ ImmutableMap.<String, String>builder().put("client-api-version",
"3");
+ Assertions.assertThatThrownBy(() -> newCatalog.initialize("nessie",
options.buildOrThrow()))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Unsupported client-api-version: 3. Can only be 1 or 2");
+ }
+ }
}
