This is an automated email from the ASF dual-hosted git repository.
yuqi4733 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 02572aa07 [#5744]feat(catalog-model): Add a basic catalog-model
framework in Gravitino (#5757)
02572aa07 is described below
commit 02572aa078405d196a268cdd98ccbefe64fd6338
Author: Jerry Shao <[email protected]>
AuthorDate: Mon Dec 9 19:43:02 2024 +0800
[#5744]feat(catalog-model): Add a basic catalog-model framework in
Gravitino (#5757)
### What changes were proposed in this pull request?
This PR adds the basic catalog-model framework in Gravitino. In the
meantime, it also makes `provider` optional for built-in catalog.
### Why are the changes needed?
Fix: #5744
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Tests will be added later on.
---
.../main/java/org/apache/gravitino/Catalog.java | 33 ++++++-
.../java/org/apache/gravitino/CatalogProvider.java | 26 ++++++
.../org/apache/gravitino/SupportsCatalogs.java | 26 +++++-
build.gradle.kts | 5 +-
catalogs/catalog-hadoop/build.gradle.kts | 9 --
.../hadoop/TestHadoopCatalogOperations.java | 10 ++
.../gravitino/catalog/hive/TestHiveCatalog.java | 5 +
.../catalog/kafka/TestKafkaCatalogOperations.java | 5 +
.../lakehouse/hudi/TestHudiCatalogOperations.java | 5 +
.../lakehouse/iceberg/TestIcebergCatalog.java | 5 +
.../lakehouse/paimon/TestPaimonCatalog.java | 5 +
catalogs/catalog-model/build.gradle.kts | 101 +++++++++++++++++++++
.../gravitino/catalog/model/ModelCatalog.java | 68 ++++++++++++++
.../catalog/model/ModelCatalogCapability.java | 29 +++---
.../model/ModelCatalogPropertiesMetadata.java | 25 ++---
.../catalog/model/ModelPropertiesMetadata.java | 25 ++---
.../model/ModelSchemaPropertiesMetadata.java | 25 ++---
.../services/org.apache.gravitino.CatalogProvider | 19 ++++
.../catalog-model/src/main/resources/model.conf | 22 +++++
clients/client-python/gravitino/api/catalog.py | 45 ++++++++-
clients/client-python/gravitino/dto/catalog_dto.py | 8 +-
.../dto/requests/catalog_create_request.py | 17 +++-
.../dto/requests/CatalogCreateRequest.java | 30 +++++-
.../dto/requests/TestCatalogCreateRequest.java | 75 +++++++++++++++
.../apache/gravitino/connector/BaseCatalog.java | 6 ++
.../gravitino/connector/HasPropertyMetadata.java | 8 ++
settings.gradle.kts | 3 +-
27 files changed, 548 insertions(+), 92 deletions(-)
diff --git a/api/src/main/java/org/apache/gravitino/Catalog.java
b/api/src/main/java/org/apache/gravitino/Catalog.java
index ce3bcc95c..7035ed948 100644
--- a/api/src/main/java/org/apache/gravitino/Catalog.java
+++ b/api/src/main/java/org/apache/gravitino/Catalog.java
@@ -40,19 +40,42 @@ public interface Catalog extends Auditable {
/** The type of the catalog. */
enum Type {
/** Catalog Type for Relational Data Structure, like db.table,
catalog.db.table. */
- RELATIONAL,
+ RELATIONAL(false),
/** Catalog Type for Fileset System (including HDFS, S3, etc.), like
path/to/file */
- FILESET,
+ FILESET(false),
/** Catalog Type for Message Queue, like Kafka://topic */
- MESSAGING,
+ MESSAGING(false),
/** Catalog Type for ML model */
- MODEL,
+ MODEL(true),
/** Catalog Type for test only. */
- UNSUPPORTED;
+ UNSUPPORTED(false);
+
+ private final boolean supportsManagedCatalog;
+
+ Type(boolean supportsManagedCatalog) {
+ this.supportsManagedCatalog = supportsManagedCatalog;
+ }
+
+ /**
+ * A flag to indicate whether the catalog type supports managed catalog.
Managed catalog is a
+ * concept in Gravitino, for the details of managed catalog, please refer
to the class comment
+ * of {@link CatalogProvider}. If the catalog type supports managed
catalog, users can create
+ * managed catalog of this type without specifying the provider, Gravitino
will use the type as
+ * the provider to create the managed catalog. If the catalog type does
not support managed
+ * catalog, users need to specify the provider when creating the catalog.
+ *
+ * <p>Currently, only the model catalog supports managed catalog.
+ *
+ * @return Whether the catalog type supports managed catalog. Returns true
if the catalog type
+ * supports managed catalog.
+ */
+ public boolean supportsManagedCatalog() {
+ return supportsManagedCatalog;
+ }
/**
* Convert the string (case-insensitive) to the catalog type.
diff --git a/api/src/main/java/org/apache/gravitino/CatalogProvider.java
b/api/src/main/java/org/apache/gravitino/CatalogProvider.java
index 429f2f9ad..e0778c1e1 100644
--- a/api/src/main/java/org/apache/gravitino/CatalogProvider.java
+++ b/api/src/main/java/org/apache/gravitino/CatalogProvider.java
@@ -18,15 +18,41 @@
*/
package org.apache.gravitino;
+import java.util.Locale;
import org.apache.gravitino.annotation.Evolving;
/**
* A Catalog provider is a class that provides a short name for a catalog.
This short name is used
* when creating a catalog.
+ *
+ * <p>There are two kinds of catalogs in Gravitino, one is managed catalog and
another is external
+ * catalog.
+ *
+ * <p>Managed catalog: A catalog and its subsidiary objects are all managed by
Gravitino. Gravitino
+ * takes care of the lifecycle of the catalog and its objects. For those
catalogs, Gravitino uses
+ * the type of the catalog as the provider short name. Note that for each
catalog type, there is
+ * only one implementation of managed catalog for that type. Currently,
Gravitino only has model
+ * catalog that is a managed catalog.
+ *
+ * <p>External catalog: A catalog its subsidiary objects are stored by an
external sources, such as
+ * Hive catalog, the DB and tables are stored in HMS. For those catalogs,
Gravitino uses a unique
+ * name as the provider short name to load the catalog. For example, Hive
catalog uses "hive" as the
+ * provider short name.
*/
@Evolving
public interface CatalogProvider {
+ /**
+ * Form the provider short name for a managed catalog. The provider short
name for a managed
+ * catalog is the catalog type in lowercase.
+ *
+ * @param type The catalog type.
+ * @return The provider short name for the managed catalog.
+ */
+ static String shortNameForManagedCatalog(Catalog.Type type) {
+ return type.name().toLowerCase(Locale.ROOT);
+ }
+
/**
* The string that represents the catalog that this provider uses. This is
overridden by children
* to provide a nice alias for the catalog.
diff --git a/api/src/main/java/org/apache/gravitino/SupportsCatalogs.java
b/api/src/main/java/org/apache/gravitino/SupportsCatalogs.java
index b10ef4010..70a291a6d 100644
--- a/api/src/main/java/org/apache/gravitino/SupportsCatalogs.java
+++ b/api/src/main/java/org/apache/gravitino/SupportsCatalogs.java
@@ -78,12 +78,15 @@ public interface SupportsCatalogs {
* Create a catalog with specified catalog name, type, provider, comment,
and properties.
*
* <p>The parameter "provider" is a short name of the catalog, used to tell
Gravitino which
- * catalog should be created. The short name should be the same as the
{@link CatalogProvider}
- * interface provided.
+ * catalog should be created. The short name:
+ *
+ * <p>1) should be the same as the {@link CatalogProvider} interface
provided. 2) can be "null" if
+ * the created catalog is the managed catalog, like model catalog. For the
details of the provider
+ * definition, see {@link CatalogProvider}.
*
* @param catalogName the name of the catalog.
* @param type the type of the catalog.
- * @param provider the provider of the catalog.
+ * @param provider the provider of the catalog, or null if the catalog is a
managed catalog.
* @param comment the comment of the catalog.
* @param properties the properties of the catalog.
* @return The created catalog.
@@ -98,6 +101,23 @@ public interface SupportsCatalogs {
Map<String, String> properties)
throws NoSuchMetalakeException, CatalogAlreadyExistsException;
+ /**
+ * Create a managed catalog with specified catalog name, type, comment, and
properties.
+ *
+ * @param catalogName the name of the catalog.
+ * @param type the type of the catalog.
+ * @param comment the comment of the catalog.
+ * @param properties the properties of the catalog.
+ * @return The created catalog.
+ * @throws NoSuchMetalakeException If the metalake does not exist.
+ * @throws CatalogAlreadyExistsException If the catalog already exists.
+ */
+ default Catalog createCatalog(
+ String catalogName, Catalog.Type type, String comment, Map<String,
String> properties)
+ throws NoSuchMetalakeException, CatalogAlreadyExistsException {
+ return createCatalog(catalogName, type, null, comment, properties);
+ }
+
/**
* Alter a catalog with specified catalog name and changes.
*
diff --git a/build.gradle.kts b/build.gradle.kts
index cc29ff4af..637e025ab 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -826,7 +826,8 @@ tasks {
":catalogs:catalog-jdbc-oceanbase:copyLibAndConfig",
":catalogs:catalog-jdbc-postgresql:copyLibAndConfig",
":catalogs:catalog-hadoop:copyLibAndConfig",
- ":catalogs:catalog-kafka:copyLibAndConfig"
+ ":catalogs:catalog-kafka:copyLibAndConfig",
+ ":catalogs:catalog-model:copyLibAndConfig"
)
}
@@ -924,7 +925,7 @@ fun printMacDockerTip() {
fun checkMacDockerConnector() {
if (!OperatingSystem.current().isMacOsX()) {
- // Only MacOs requires the use of `docker-connector`
+ // Only macOS requires the use of `docker-connector`
return
}
diff --git a/catalogs/catalog-hadoop/build.gradle.kts
b/catalogs/catalog-hadoop/build.gradle.kts
index 409a87fb1..84488efb0 100644
--- a/catalogs/catalog-hadoop/build.gradle.kts
+++ b/catalogs/catalog-hadoop/build.gradle.kts
@@ -151,15 +151,6 @@ tasks {
}
tasks.test {
- doFirst {
- val testMode = project.properties["testMode"] as? String ?: "embedded"
- if (testMode == "deploy") {
- environment("GRAVITINO_HOME", project.rootDir.path +
"/distribution/package")
- } else if (testMode == "embedded") {
- environment("GRAVITINO_HOME", project.rootDir.path)
- }
- }
-
val skipITs = project.hasProperty("skipITs")
if (skipITs) {
// Exclude integration tests
diff --git
a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java
b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java
index 3c1ea4ff0..cff6c3489 100644
---
a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java
+++
b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java
@@ -131,6 +131,11 @@ public class TestHadoopCatalogOperations {
public PropertiesMetadata topicPropertiesMetadata() throws
UnsupportedOperationException {
throw new UnsupportedOperationException("Does not support topic
properties");
}
+
+ @Override
+ public PropertiesMetadata modelPropertiesMetadata() throws
UnsupportedOperationException {
+ throw new UnsupportedOperationException("Does not support model
properties");
+ }
};
private static EntityStore store;
@@ -733,6 +738,11 @@ public class TestHadoopCatalogOperations {
public PropertiesMetadata topicPropertiesMetadata() throws
UnsupportedOperationException {
return null;
}
+
+ @Override
+ public PropertiesMetadata modelPropertiesMetadata() throws
UnsupportedOperationException {
+ return null;
+ }
};
try {
diff --git
a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalog.java
b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalog.java
index cb1d8c777..7b3f944b9 100644
---
a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalog.java
+++
b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalog.java
@@ -66,6 +66,11 @@ public class TestHiveCatalog extends
MiniHiveMetastoreService {
public PropertiesMetadata topicPropertiesMetadata() throws
UnsupportedOperationException {
throw new UnsupportedOperationException("Topic properties are not
supported");
}
+
+ @Override
+ public PropertiesMetadata modelPropertiesMetadata() throws
UnsupportedOperationException {
+ throw new UnsupportedOperationException("Model properties are not
supported");
+ }
};
@Test
diff --git
a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java
b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java
index 58cd55042..90451776d 100644
---
a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java
+++
b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java
@@ -117,6 +117,11 @@ public class TestKafkaCatalogOperations extends
KafkaClusterEmbedded {
public PropertiesMetadata topicPropertiesMetadata() throws
UnsupportedOperationException {
return TOPIC_PROPERTIES_METADATA;
}
+
+ @Override
+ public PropertiesMetadata modelPropertiesMetadata() throws
UnsupportedOperationException {
+ throw new UnsupportedOperationException("Not supported");
+ }
};
private static EntityStore store;
private static IdGenerator idGenerator;
diff --git
a/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/TestHudiCatalogOperations.java
b/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/TestHudiCatalogOperations.java
index 01e616647..a4b86c0c5 100644
---
a/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/TestHudiCatalogOperations.java
+++
b/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/TestHudiCatalogOperations.java
@@ -59,6 +59,11 @@ public class TestHudiCatalogOperations {
public PropertiesMetadata topicPropertiesMetadata() throws
UnsupportedOperationException {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public PropertiesMetadata modelPropertiesMetadata() throws
UnsupportedOperationException {
+ throw new UnsupportedOperationException();
+ }
};
@Test
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java
index 36c2abbfd..5c6571972 100644
---
a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java
+++
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java
@@ -64,6 +64,11 @@ public class TestIcebergCatalog {
public PropertiesMetadata topicPropertiesMetadata() throws
UnsupportedOperationException {
throw new UnsupportedOperationException("Topic properties are not
supported");
}
+
+ @Override
+ public PropertiesMetadata modelPropertiesMetadata() throws
UnsupportedOperationException {
+ throw new UnsupportedOperationException("Model properties are not
supported");
+ }
};
@Test
diff --git
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java
index e6439a8d7..010b6daee 100644
---
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java
+++
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java
@@ -72,6 +72,11 @@ public class TestPaimonCatalog {
public PropertiesMetadata topicPropertiesMetadata() throws
UnsupportedOperationException {
throw new UnsupportedOperationException("Topic properties are not
supported");
}
+
+ @Override
+ public PropertiesMetadata modelPropertiesMetadata() throws
UnsupportedOperationException {
+ throw new UnsupportedOperationException("Model properties are not
supported");
+ }
};
private static String tempDir =
diff --git a/catalogs/catalog-model/build.gradle.kts
b/catalogs/catalog-model/build.gradle.kts
new file mode 100644
index 000000000..33f8413a3
--- /dev/null
+++ b/catalogs/catalog-model/build.gradle.kts
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+description = "catalog-model"
+
+plugins {
+ `maven-publish`
+ id("java")
+ id("idea")
+}
+
+dependencies {
+ implementation(project(":api")) {
+ exclude(group = "*")
+ }
+
+ implementation(project(":core")) {
+ exclude(group = "*")
+ }
+ implementation(project(":common")) {
+ exclude(group = "*")
+ }
+
+ implementation(project(":catalogs:catalog-common")) {
+ exclude(group = "*")
+ }
+
+ compileOnly(libs.guava)
+
+ implementation(libs.slf4j.api)
+
+ testImplementation(libs.bundles.log4j)
+ testImplementation(libs.mockito.core)
+ testImplementation(libs.mockito.inline)
+ testImplementation(libs.junit.jupiter.api)
+ testImplementation(libs.junit.jupiter.params)
+
+ testRuntimeOnly(libs.junit.jupiter.engine)
+}
+
+tasks {
+ val runtimeJars by registering(Copy::class) {
+ from(configurations.runtimeClasspath)
+ into("build/libs")
+ }
+
+ val copyCatalogLibs by registering(Copy::class) {
+ dependsOn("jar", "runtimeJars")
+ from("build/libs") {
+ exclude("slf4j-*.jar")
+ exclude("guava-*.jar")
+ }
+ into("$rootDir/distribution/package/catalogs/model/libs")
+ }
+
+ val copyCatalogConfig by registering(Copy::class) {
+ from("src/main/resources")
+ into("$rootDir/distribution/package/catalogs/model/conf")
+
+ include("model.conf")
+
+ exclude { details ->
+ details.file.isDirectory()
+ }
+
+ fileMode = 0b111101101
+ }
+
+ register("copyLibAndConfig", Copy::class) {
+ dependsOn(copyCatalogConfig, copyCatalogLibs)
+ }
+}
+
+tasks.test {
+ val skipITs = project.hasProperty("skipITs")
+ if (skipITs) {
+ // Exclude integration tests
+ exclude("**/integration/test/**")
+ } else {
+ dependsOn(tasks.jar)
+ }
+}
+
+tasks.getByName("generateMetadataFileForMavenJavaPublication") {
+ dependsOn("runtimeJars")
+}
diff --git
a/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalog.java
b/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalog.java
new file mode 100644
index 000000000..51951d440
--- /dev/null
+++
b/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalog.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.model;
+
+import java.util.Map;
+import org.apache.gravitino.CatalogProvider;
+import org.apache.gravitino.connector.BaseCatalog;
+import org.apache.gravitino.connector.CatalogOperations;
+import org.apache.gravitino.connector.PropertiesMetadata;
+import org.apache.gravitino.connector.capability.Capability;
+
+public class ModelCatalog extends BaseCatalog<ModelCatalog> {
+
+ private static final ModelCatalogPropertiesMetadata CATALOG_PROPERTIES_META =
+ new ModelCatalogPropertiesMetadata();
+
+ private static final ModelSchemaPropertiesMetadata SCHEMA_PROPERTIES_META =
+ new ModelSchemaPropertiesMetadata();
+
+ private static final ModelPropertiesMetadata MODEL_PROPERTIES_META =
+ new ModelPropertiesMetadata();
+
+ @Override
+ public String shortName() {
+ return CatalogProvider.shortNameForManagedCatalog(super.type());
+ }
+
+ @Override
+ protected CatalogOperations newOps(Map<String, String> config) {
+ return null;
+ }
+
+ @Override
+ public PropertiesMetadata catalogPropertiesMetadata() throws
UnsupportedOperationException {
+ return CATALOG_PROPERTIES_META;
+ }
+
+ @Override
+ public PropertiesMetadata schemaPropertiesMetadata() throws
UnsupportedOperationException {
+ return SCHEMA_PROPERTIES_META;
+ }
+
+ @Override
+ public PropertiesMetadata modelPropertiesMetadata() throws
UnsupportedOperationException {
+ return MODEL_PROPERTIES_META;
+ }
+
+ @Override
+ protected Capability newCapability() {
+ return new ModelCatalogCapability();
+ }
+}
diff --git a/api/src/main/java/org/apache/gravitino/CatalogProvider.java
b/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogCapability.java
similarity index 58%
copy from api/src/main/java/org/apache/gravitino/CatalogProvider.java
copy to
catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogCapability.java
index 429f2f9ad..b19c9ba2e 100644
--- a/api/src/main/java/org/apache/gravitino/CatalogProvider.java
+++
b/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogCapability.java
@@ -16,22 +16,19 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.gravitino;
+package org.apache.gravitino.catalog.model;
-import org.apache.gravitino.annotation.Evolving;
+import java.util.Objects;
+import org.apache.gravitino.connector.capability.Capability;
+import org.apache.gravitino.connector.capability.CapabilityResult;
-/**
- * A Catalog provider is a class that provides a short name for a catalog.
This short name is used
- * when creating a catalog.
- */
-@Evolving
-public interface CatalogProvider {
-
- /**
- * The string that represents the catalog that this provider uses. This is
overridden by children
- * to provide a nice alias for the catalog.
- *
- * @return The string that represents the catalog that this provider uses.
- */
- String shortName();
+public class ModelCatalogCapability implements Capability {
+ @Override
+ public CapabilityResult managedStorage(Scope scope) {
+ if (Objects.requireNonNull(scope) == Scope.SCHEMA) {
+ return CapabilityResult.SUPPORTED;
+ }
+ return CapabilityResult.unsupported(
+ String.format("Model catalog does not support managed storage for
%s.", scope));
+ }
}
diff --git a/api/src/main/java/org/apache/gravitino/CatalogProvider.java
b/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogPropertiesMetadata.java
similarity index 60%
copy from api/src/main/java/org/apache/gravitino/CatalogProvider.java
copy to
catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogPropertiesMetadata.java
index 429f2f9ad..2ae510dbf 100644
--- a/api/src/main/java/org/apache/gravitino/CatalogProvider.java
+++
b/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogPropertiesMetadata.java
@@ -16,22 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.gravitino;
+package org.apache.gravitino.catalog.model;
-import org.apache.gravitino.annotation.Evolving;
+import java.util.Collections;
+import java.util.Map;
+import org.apache.gravitino.connector.BaseCatalogPropertiesMetadata;
+import org.apache.gravitino.connector.PropertyEntry;
-/**
- * A Catalog provider is a class that provides a short name for a catalog.
This short name is used
- * when creating a catalog.
- */
-@Evolving
-public interface CatalogProvider {
+public class ModelCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetadata {
- /**
- * The string that represents the catalog that this provider uses. This is
overridden by children
- * to provide a nice alias for the catalog.
- *
- * @return The string that represents the catalog that this provider uses.
- */
- String shortName();
+ @Override
+ protected Map<String, PropertyEntry<?>> specificPropertyEntries() {
+ return Collections.emptyMap();
+ }
}
diff --git a/api/src/main/java/org/apache/gravitino/CatalogProvider.java
b/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelPropertiesMetadata.java
similarity index 60%
copy from api/src/main/java/org/apache/gravitino/CatalogProvider.java
copy to
catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelPropertiesMetadata.java
index 429f2f9ad..67bd58e76 100644
--- a/api/src/main/java/org/apache/gravitino/CatalogProvider.java
+++
b/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelPropertiesMetadata.java
@@ -16,22 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.gravitino;
+package org.apache.gravitino.catalog.model;
-import org.apache.gravitino.annotation.Evolving;
+import java.util.Collections;
+import java.util.Map;
+import org.apache.gravitino.connector.BasePropertiesMetadata;
+import org.apache.gravitino.connector.PropertyEntry;
-/**
- * A Catalog provider is a class that provides a short name for a catalog.
This short name is used
- * when creating a catalog.
- */
-@Evolving
-public interface CatalogProvider {
+public class ModelPropertiesMetadata extends BasePropertiesMetadata {
- /**
- * The string that represents the catalog that this provider uses. This is
overridden by children
- * to provide a nice alias for the catalog.
- *
- * @return The string that represents the catalog that this provider uses.
- */
- String shortName();
+ @Override
+ protected Map<String, PropertyEntry<?>> specificPropertyEntries() {
+ return Collections.emptyMap();
+ }
}
diff --git a/api/src/main/java/org/apache/gravitino/CatalogProvider.java
b/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelSchemaPropertiesMetadata.java
similarity index 60%
copy from api/src/main/java/org/apache/gravitino/CatalogProvider.java
copy to
catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelSchemaPropertiesMetadata.java
index 429f2f9ad..35fb2671c 100644
--- a/api/src/main/java/org/apache/gravitino/CatalogProvider.java
+++
b/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelSchemaPropertiesMetadata.java
@@ -16,22 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.gravitino;
+package org.apache.gravitino.catalog.model;
-import org.apache.gravitino.annotation.Evolving;
+import java.util.Collections;
+import java.util.Map;
+import org.apache.gravitino.connector.BasePropertiesMetadata;
+import org.apache.gravitino.connector.PropertyEntry;
-/**
- * A Catalog provider is a class that provides a short name for a catalog.
This short name is used
- * when creating a catalog.
- */
-@Evolving
-public interface CatalogProvider {
+public class ModelSchemaPropertiesMetadata extends BasePropertiesMetadata {
- /**
- * The string that represents the catalog that this provider uses. This is
overridden by children
- * to provide a nice alias for the catalog.
- *
- * @return The string that represents the catalog that this provider uses.
- */
- String shortName();
+ @Override
+ protected Map<String, PropertyEntry<?>> specificPropertyEntries() {
+ return Collections.emptyMap();
+ }
}
diff --git
a/catalogs/catalog-model/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider
b/catalogs/catalog-model/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider
new file mode 100644
index 000000000..37c682aa7
--- /dev/null
+++
b/catalogs/catalog-model/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+org.apache.gravitino.catalog.model.ModelCatalog
diff --git a/catalogs/catalog-model/src/main/resources/model.conf
b/catalogs/catalog-model/src/main/resources/model.conf
new file mode 100644
index 000000000..0f98b092a
--- /dev/null
+++ b/catalogs/catalog-model/src/main/resources/model.conf
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# This file holds common properties for model catalog. All the created model
catalog will
+# leverage this conf file as default configuration. In the meantime, user
could specify catalog
+# properties to override the default configuration.
diff --git a/clients/client-python/gravitino/api/catalog.py
b/clients/client-python/gravitino/api/catalog.py
index 81f8d56f9..3ad137f8c 100644
--- a/clients/client-python/gravitino/api/catalog.py
+++ b/clients/client-python/gravitino/api/catalog.py
@@ -31,18 +31,55 @@ class Catalog(Auditable):
class Type(Enum):
"""The type of the catalog."""
- RELATIONAL = "relational"
+ RELATIONAL = ("relational", False)
""""Catalog Type for Relational Data Structure, like db.table,
catalog.db.table."""
- FILESET = "fileset"
+ FILESET = ("fileset", False)
"""Catalog Type for Fileset System (including HDFS, S3, etc.), like
path/to/file"""
- MESSAGING = "messaging"
+ MESSAGING = ("messaging", False)
"""Catalog Type for Message Queue, like kafka://topic"""
- UNSUPPORTED = "unsupported"
+ MODEL = ("model", True)
+ """Catalog Type for ML model"""
+
+ UNSUPPORTED = ("unsupported", False)
"""Catalog Type for test only."""
+ def __init__(self, type_name, supports_managed_catalog):
+ self._type_name = type_name
+ self._supports_managed_catalog = supports_managed_catalog
+
+ @classmethod
+ def type_serialize(cls, catalog_type):
+ return catalog_type.type_name
+
+ @classmethod
+ def type_deserialize(cls, type_name):
+ for member in cls:
+ if member.type_name == type_name:
+ return member
+ return cls.UNSUPPORTED
+
+ @property
+ def supports_managed_catalog(self):
+ """
+ A flag to indicate if the catalog type supports managed catalog.
Managed catalog is a
+ concept in Gravitino, which means Gravitino will manage the
lifecycle of the catalog
+ and its subsidiaries. If the catalog type supports managed
catalog, users can create
+ managed catalog of this type without specifying the catalog
provider, Gravitino will
+ use the type as the provider to create the managed catalog. If the
catalog type does
+ not support managed catalog, users need to specify the provider to
create the catalog.
+ """
+ return self._supports_managed_catalog
+
+ @property
+ def type_name(self):
+ """
+ The name of the catalog type.
+ """
+ return self._type_name
+
PROPERTY_PACKAGE = "package"
"""A reserved property to specify the package location of the catalog. The
"package" is a string
of path to the folder where all the catalog related dependencies is
located. The dependencies
diff --git a/clients/client-python/gravitino/dto/catalog_dto.py
b/clients/client-python/gravitino/dto/catalog_dto.py
index 0ce25c19b..a0167b176 100644
--- a/clients/client-python/gravitino/dto/catalog_dto.py
+++ b/clients/client-python/gravitino/dto/catalog_dto.py
@@ -29,7 +29,13 @@ class CatalogDTO(Catalog):
"""Data transfer object representing catalog information."""
_name: str = field(metadata=config(field_name="name"))
- _type: Catalog.Type = field(metadata=config(field_name="type"))
+ _type: Catalog.Type = field(
+ metadata=config(
+ field_name="type",
+ encoder=Catalog.Type.type_serialize,
+ decoder=Catalog.Type.type_deserialize,
+ )
+ )
_provider: str = field(metadata=config(field_name="provider"))
_comment: str = field(metadata=config(field_name="comment"))
_properties: Dict[str, str] =
field(metadata=config(field_name="properties"))
diff --git
a/clients/client-python/gravitino/dto/requests/catalog_create_request.py
b/clients/client-python/gravitino/dto/requests/catalog_create_request.py
index f787b5b83..885011118 100644
--- a/clients/client-python/gravitino/dto/requests/catalog_create_request.py
+++ b/clients/client-python/gravitino/dto/requests/catalog_create_request.py
@@ -29,8 +29,14 @@ class CatalogCreateRequest(RESTRequest):
"""Represents a request to create a catalog."""
_name: str = field(metadata=config(field_name="name"))
- _type: Catalog.Type = field(metadata=config(field_name="type"))
- _provider: str = field(metadata=config(field_name="provider"))
+ _type: Catalog.Type = field(
+ metadata=config(
+ field_name="type",
+ encoder=Catalog.Type.type_serialize,
+ decoder=Catalog.Type.type_deserialize,
+ )
+ )
+ _provider: Optional[str] = field(metadata=config(field_name="provider"))
_comment: Optional[str] = field(metadata=config(field_name="comment"))
_properties: Optional[Dict[str, str]] = field(
metadata=config(field_name="properties")
@@ -60,5 +66,8 @@ class CatalogCreateRequest(RESTRequest):
raise ValueError('"name" field is required and cannot be empty')
if not self._type:
raise ValueError('"catalog_type" field is required and cannot be
empty')
- if not self._provider:
- raise ValueError('"provider" field is required and cannot be
empty')
+ if not self._provider and not self._type.supports_managed_catalog:
+ raise ValueError(
+ '"provider" field is required and cannot be empty for catalog
type '
+ "that does not support managed catalog"
+ )
diff --git
a/common/src/main/java/org/apache/gravitino/dto/requests/CatalogCreateRequest.java
b/common/src/main/java/org/apache/gravitino/dto/requests/CatalogCreateRequest.java
index 0622540ad..3da657967 100644
---
a/common/src/main/java/org/apache/gravitino/dto/requests/CatalogCreateRequest.java
+++
b/common/src/main/java/org/apache/gravitino/dto/requests/CatalogCreateRequest.java
@@ -19,6 +19,7 @@
package org.apache.gravitino.dto.requests;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonSetter;
import com.google.common.base.Preconditions;
import java.util.Map;
import javax.annotation.Nullable;
@@ -27,6 +28,7 @@ import lombok.Getter;
import lombok.ToString;
import org.apache.commons.lang3.StringUtils;
import org.apache.gravitino.Catalog;
+import org.apache.gravitino.CatalogProvider;
import org.apache.gravitino.rest.RESTRequest;
/** Represents a request to create a catalog. */
@@ -42,7 +44,7 @@ public class CatalogCreateRequest implements RESTRequest {
private final Catalog.Type type;
@JsonProperty("provider")
- private final String provider;
+ private String provider;
@Nullable
@JsonProperty("comment")
@@ -79,6 +81,27 @@ public class CatalogCreateRequest implements RESTRequest {
this.properties = properties;
}
+ /**
+ * Sets the provider of the catalog if it is null. The value of provider in
the request can be
+ * null if the catalog is a managed catalog. For such request, the value
will be set when it is
+ * deserialized.
+ *
+ * @param provider The provider of the catalog.
+ */
+ @JsonSetter(value = "provider")
+ public void setProvider(String provider) {
+ if (StringUtils.isNotBlank(provider)) {
+ this.provider = provider;
+ } else if (type != null && type.supportsManagedCatalog()) {
+ this.provider = CatalogProvider.shortNameForManagedCatalog(type);
+ } else {
+ throw new IllegalStateException(
+ "Provider cannot be null for catalog type "
+ + type
+ + " that doesn't support managed catalog");
+ }
+ }
+
/**
* Validates the fields of the request.
*
@@ -90,6 +113,9 @@ public class CatalogCreateRequest implements RESTRequest {
StringUtils.isNotBlank(name), "\"name\" field is required and cannot
be empty");
Preconditions.checkArgument(type != null, "\"type\" field is required and
cannot be empty");
Preconditions.checkArgument(
- StringUtils.isNotBlank(provider), "\"provider\" field is required and
cannot be empty");
+ StringUtils.isNotBlank(provider) || type.supportsManagedCatalog(),
+ "\"provider\" field is required and cannot be empty for catalog type "
+ + type
+ + " that doesn't support managed catalog");
}
}
diff --git
a/common/src/test/java/org/apache/gravitino/dto/requests/TestCatalogCreateRequest.java
b/common/src/test/java/org/apache/gravitino/dto/requests/TestCatalogCreateRequest.java
new file mode 100644
index 000000000..b4b7383a7
--- /dev/null
+++
b/common/src/test/java/org/apache/gravitino/dto/requests/TestCatalogCreateRequest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.dto.requests;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.google.common.collect.ImmutableMap;
+import java.util.Locale;
+import org.apache.gravitino.Catalog;
+import org.apache.gravitino.json.JsonUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestCatalogCreateRequest {
+
+ @Test
+ public void testCatalogCreateRequestSerDe() throws JsonProcessingException {
+ CatalogCreateRequest request =
+ new CatalogCreateRequest(
+ "catalog_test",
+ Catalog.Type.MODEL,
+ "provider_test",
+ "catalog comment",
+ ImmutableMap.of("key", "value"));
+
+ String serJson = JsonUtils.objectMapper().writeValueAsString(request);
+ CatalogCreateRequest deserRequest =
+ JsonUtils.objectMapper().readValue(serJson,
CatalogCreateRequest.class);
+
+ Assertions.assertEquals(request, deserRequest);
+ Assertions.assertEquals("catalog_test", deserRequest.getName());
+ Assertions.assertEquals(Catalog.Type.MODEL, deserRequest.getType());
+ Assertions.assertEquals("provider_test", deserRequest.getProvider());
+ Assertions.assertEquals("catalog comment", deserRequest.getComment());
+ Assertions.assertEquals(ImmutableMap.of("key", "value"),
deserRequest.getProperties());
+
+ // Test with null provider, comment and properties
+ CatalogCreateRequest request1 =
+ new CatalogCreateRequest("catalog_test", Catalog.Type.MODEL, null,
null, null);
+
+ String serJson1 = JsonUtils.objectMapper().writeValueAsString(request1);
+ CatalogCreateRequest deserRequest1 =
+ JsonUtils.objectMapper().readValue(serJson1,
CatalogCreateRequest.class);
+
+ Assertions.assertEquals(
+ deserRequest1.getType().name().toLowerCase(Locale.ROOT),
deserRequest1.getProvider());
+ Assertions.assertNull(deserRequest1.getComment());
+ Assertions.assertNull(deserRequest1.getProperties());
+
+ // Test using null provider with catalog type doesn't support managed
catalog
+ CatalogCreateRequest request2 =
+ new CatalogCreateRequest("catalog_test", Catalog.Type.RELATIONAL,
null, null, null);
+
+ String serJson2 = JsonUtils.objectMapper().writeValueAsString(request2);
+ Assertions.assertThrows(
+ JsonMappingException.class,
+ () -> JsonUtils.objectMapper().readValue(serJson2,
CatalogCreateRequest.class));
+ }
+}
diff --git a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java
b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java
index 07bc83b62..dbc9c0859 100644
--- a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java
+++ b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java
@@ -151,6 +151,12 @@ public abstract class BaseCatalog<T extends BaseCatalog>
"The catalog does not support topic properties metadata");
}
+ @Override
+ public PropertiesMetadata modelPropertiesMetadata() throws
UnsupportedOperationException {
+ throw new UnsupportedOperationException(
+ "The catalog does not support model properties metadata");
+ }
+
/**
* Retrieves the CatalogOperations instance associated with this catalog.
Lazily initializes the
* instance if not already created.
diff --git
a/core/src/main/java/org/apache/gravitino/connector/HasPropertyMetadata.java
b/core/src/main/java/org/apache/gravitino/connector/HasPropertyMetadata.java
index b918bd01c..e8e02b6ff 100644
--- a/core/src/main/java/org/apache/gravitino/connector/HasPropertyMetadata.java
+++ b/core/src/main/java/org/apache/gravitino/connector/HasPropertyMetadata.java
@@ -63,4 +63,12 @@ public interface HasPropertyMetadata {
* @throws UnsupportedOperationException if the entity does not support
topic properties.
*/
PropertiesMetadata topicPropertiesMetadata() throws
UnsupportedOperationException;
+
+ /**
+ * Returns the model property metadata.
+ *
+ * @return The model property metadata.
+ * @throws UnsupportedOperationException if the entity does not support
model properties.
+ */
+ PropertiesMetadata modelPropertiesMetadata() throws
UnsupportedOperationException;
}
diff --git a/settings.gradle.kts b/settings.gradle.kts
index a36fde93c..6822edcad 100644
--- a/settings.gradle.kts
+++ b/settings.gradle.kts
@@ -41,6 +41,7 @@ include(
)
include("catalogs:catalog-hadoop")
include("catalogs:catalog-kafka")
+include("catalogs:catalog-model")
include(
"clients:client-java",
"clients:client-java-runtime",
@@ -49,7 +50,7 @@ include(
"clients:client-python",
"clients:cli"
)
-if (gradle.startParameter.projectProperties["enableFuse"]?.toBoolean() ?:
false) {
+if (gradle.startParameter.projectProperties["enableFuse"]?.toBoolean() ==
true) {
include("clients:filesystem-fuse")
} else {
println("Skipping filesystem-fuse module since enableFuse is set to false")
