This is an automated email from the ASF dual-hosted git repository.
dhuo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 34dbfcea1 API Spec: Add ConnectionConfigInfo to ExternalCatalog (#1026)
34dbfcea1 is described below
commit 34dbfcea164d85df95784f2202c56b3ff6c77231
Author: Dennis Huo <[email protected]>
AuthorDate: Wed Mar 26 18:02:53 2025 -0700
API Spec: Add ConnectionConfigInfo to ExternalCatalog (#1026)
* API Spec: Add ConnectionConfigInfo to ExternalCatalog
Remove the currently unused remoteUrl field from the top-level
ExternalCatalog
into the ConnectionConfigInfo as "uri" instead for better consistency;
remote catalogs in the future may be defined by arbitrary URIs that are not,
for example, http(s) URLs.
This is just the spec definition for now, so it's not yet wired into the
internal entity layer or persistence objects.
Allow extensibility of different connection types in the future even if we
start with only an ICEBERG_REST type.
Similarly, provide extensibility for different authn mechanisms to use
with the connection.
---
.../it/test/PolarisApplicationIntegrationTest.java | 1 -
.../PolarisManagementServiceIntegrationTest.java | 3 -
.../it/test/PolarisSparkIntegrationTest.java | 1 -
.../apache/polaris/core/entity/CatalogEntity.java | 5 --
spec/polaris-management-service.yml | 90 +++++++++++++++++++++-
5 files changed, 87 insertions(+), 13 deletions(-)
diff --git
a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java
b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java
index 0be4312c0..2437601e0 100644
---
a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java
+++
b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java
@@ -203,7 +203,6 @@ public class PolarisApplicationIntegrationTest {
.setStorageConfigInfo(storageConfig)
.build()
: ExternalCatalog.builder()
- .setRemoteUrl("http://faraway.com";)
.setName(catalogName)
.setType(catalogType)
.setProperties(props)
diff --git
a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java
b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java
index d743351d9..573623021 100644
---
a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java
+++
b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java
@@ -466,12 +466,10 @@ public class PolarisManagementServiceIntegrationTest {
.setAllowedLocations(List.of("s3://my-old-bucket/path/to/data"))
.build();
String catalogName = client.newEntityName("my-external-catalog");
- String remoteUrl = "http://localhost:8080";;
Catalog catalog =
ExternalCatalog.builder()
.setType(Catalog.TypeEnum.EXTERNAL)
.setName(catalogName)
- .setRemoteUrl(remoteUrl)
.setProperties(new
CatalogProperties("s3://my-bucket/path/to/data"))
.setStorageConfigInfo(awsConfigModel)
.build();
@@ -484,7 +482,6 @@ public class PolarisManagementServiceIntegrationTest {
.isNotNull()
.isInstanceOf(ExternalCatalog.class)
.asInstanceOf(InstanceOfAssertFactories.type(ExternalCatalog.class))
- .returns(remoteUrl, ExternalCatalog::getRemoteUrl)
.extracting(ExternalCatalog::getStorageConfigInfo)
.isNotNull()
.isInstanceOf(AwsStorageConfigInfo.class)
diff --git
a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java
b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java
index 7333baa39..10868c1ea 100644
---
a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java
+++
b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java
@@ -173,7 +173,6 @@ public class PolarisSparkIntegrationTest {
.setName(externalCatalogName)
.setProperties(externalProps)
.setStorageConfigInfo(awsConfigModel)
- .setRemoteUrl("http://dummy_url";)
.build();
managementApi.createCatalog(externalCatalog);
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java
b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java
index dbc31c0ff..a49bba9e6 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java
@@ -65,7 +65,6 @@ public class CatalogEntity extends PolarisEntity {
// translated into "s3://my-bucket/base/location/ns1/ns2/table1".
public static final String
REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY =
"replace-new-location-prefix-with-catalog-default";
- public static final String REMOTE_URL = "remoteUrl";
public CatalogEntity(PolarisBaseEntity sourceEntity) {
super(sourceEntity);
@@ -86,9 +85,6 @@ public class CatalogEntity extends PolarisEntity {
.setProperties(catalog.getProperties().toMap())
.setCatalogType(catalog.getType().name());
Map<String, String> internalProperties = new HashMap<>();
- if (catalog instanceof ExternalCatalog) {
- internalProperties.put(REMOTE_URL, ((ExternalCatalog)
catalog).getRemoteUrl());
- }
internalProperties.put(CATALOG_TYPE_PROPERTY, catalog.getType().name());
builder.setInternalProperties(internalProperties);
builder.setStorageConfigurationInfo(
@@ -120,7 +116,6 @@ public class CatalogEntity extends PolarisEntity {
: ExternalCatalog.builder()
.setType(Catalog.TypeEnum.EXTERNAL)
.setName(getName())
- .setRemoteUrl(getInternalPropertiesAsMap().get(REMOTE_URL))
.setProperties(catalogProps)
.setCreateTimestamp(getCreateTimestamp())
.setLastUpdateTimestamp(getLastUpdateTimestamp())
diff --git a/spec/polaris-management-service.yml
b/spec/polaris-management-service.yml
index a58f5a69b..318f17a6c 100644
--- a/spec/polaris-management-service.yml
+++ b/spec/polaris-management-service.yml
@@ -850,9 +850,93 @@ components:
- $ref: "#/components/schemas/Catalog"
- type: object
properties:
- remoteUrl:
- type: string
- description: URL to the remote catalog API
+ connectionConfigInfo:
+ $ref: "#/components/schemas/ConnectionConfigInfo"
+
+ ConnectionConfigInfo:
+ type: object
+ description: A connection configuration representing a remote catalog
service. IMPORTANT - Specifying a
+ ConnectionConfigInfo in an ExternalCatalog is currently an
experimental API and is subject to change.
+ properties:
+ connectionType:
+ type: string
+ enum:
+ - ICEBERG_REST
+ description: The type of remote catalog service represented by this
connection
+ uri:
+ type: string
+ description: URI to the remote catalog service
+ authenticationParameters:
+ $ref: "#/components/schemas/AuthenticationParameters"
+ required:
+ - connectionType
+ discriminator:
+ propertyName: connectionType
+ mapping:
+ ICEBERG_REST: "#/components/schemas/IcebergRestConnectionConfigInfo"
+
+ IcebergRestConnectionConfigInfo:
+ type: object
+ description: Configuration necessary for connecting to an Iceberg REST
Catalog
+ allOf:
+ - $ref: '#/components/schemas/ConnectionConfigInfo'
+ properties:
+ remoteCatalogName:
+ type: string
+ description: The name of a remote catalog instance within the remote
catalog service; in some older systems
+ this is specified as the 'warehouse' when multiple logical
catalogs are served under the same base
+ uri, and often translates into a 'prefix' added to all REST
resource paths
+
+ AuthenticationParameters:
+ type: object
+ description: Authentication-specific information for a REST connection
+ properties:
+ authenticationType:
+ type: string
+ enum:
+ - OAUTH
+ - BEARER
+ description: The type of authentication to use when connecting to
the remote rest service
+ required:
+ - authenticationType
+ discriminator:
+ propertyName: authenticationType
+ mapping:
+ OAUTH: "#/components/schemas/OAuthClientCredentialsParameters"
+ BEARER: "#/components/schemas/BearerAuthenticationParameters"
+
+ OAuthClientCredentialsParameters:
+ type: object
+ description: OAuth authentication based on client_id/client_secret
+ allOf:
+ - $ref: '#/components/schemas/AuthenticationParameters'
+ properties:
+ tokenUri:
+ type: string
+ description: Token server URI
+ clientId:
+ type: string
+ description: oauth client id
+ clientSecret:
+ type: string
+ format: password
+ description: oauth client secret (input-only)
+ scopes:
+ type: array
+ items:
+ type: string
+ description: oauth scopes to specify when exchanging for a
short-lived access token
+
+ BearerAuthenticationParameters:
+ type: object
+ description: Bearer authentication directly embedded in request auth
headers
+ allOf:
+ - $ref: '#/components/schemas/AuthenticationParameters'
+ properties:
+ bearerToken:
+ type: string
+ format: password
+ description: Bearer token (input-only)
StorageConfigInfo:
type: object
