This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 5d00b538d5 [#6750] improvement(core): property framework supports
prefix (#6751)
5d00b538d5 is described below
commit 5d00b538d5e8c4079c200889c06439627392fcc4
Author: mchades <[email protected]>
AuthorDate: Thu Apr 3 13:02:58 2025 +0800
[#6750] improvement(core): property framework supports prefix (#6751)
### What changes were proposed in this pull request?
property framework supports prefix
### Why are the changes needed?
some properties with fixed prefixes maybe immutable or required and need
to be verified.
Fix: #6750
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
tests added
---
.../hadoop/HadoopFilesetPropertiesMetadata.java | 10 ++
.../hadoop/integration/test/HadoopCatalogIT.java | 27 ++++
.../gravitino/catalog/hive/TestHiveCatalog.java | 4 +-
.../kafka/integration/test/CatalogKafkaIT.java | 6 +-
.../hudi/integration/test/HudiCatalogHMSIT.java | 16 +-
.../catalog/PropertiesMetadataHelpers.java | 34 ++--
.../gravitino/connector/PropertiesMetadata.java | 105 ++++++++++--
.../apache/gravitino/connector/PropertyEntry.java | 109 ++++++++++++-
.../java/org/apache/gravitino/TestCatalog.java | 86 ++++++++--
.../gravitino/catalog/TestCatalogManager.java | 178 +++++++++++++++++----
.../catalog/TestCatalogNormalizeDispatcher.java | 12 +-
.../catalog/TestFilesetOperationDispatcher.java | 4 +-
.../catalog/TestModelOperationDispatcher.java | 4 +-
.../gravitino/catalog/TestOperationDispatcher.java | 7 +-
.../catalog/TestSchemaOperationDispatcher.java | 4 +-
.../catalog/TestTableOperationDispatcher.java | 4 +-
.../catalog/TestTopicOperationDispatcher.java | 9 +-
docs/hadoop-catalog.md | 26 +--
18 files changed, 536 insertions(+), 109 deletions(-)
diff --git
a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java
b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java
index 1bdf224671..2a43de62e1 100644
---
a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java
+++
b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java
@@ -18,6 +18,7 @@
*/
package org.apache.gravitino.catalog.hadoop;
+import static org.apache.gravitino.file.Fileset.LOCATION_PLACEHOLDER_PREFIX;
import static org.apache.gravitino.file.Fileset.RESERVED_CATALOG_PLACEHOLDER;
import static org.apache.gravitino.file.Fileset.RESERVED_FILESET_PLACEHOLDER;
import static org.apache.gravitino.file.Fileset.RESERVED_SCHEMA_PLACEHOLDER;
@@ -54,6 +55,15 @@ public class HadoopFilesetPropertiesMetadata extends
BasePropertiesMetadata {
RESERVED_FILESET_PLACEHOLDER,
"The placeholder will be replaced to fileset name in the
location",
true /* hidden */))
+ .put(
+ LOCATION_PLACEHOLDER_PREFIX,
+ PropertyEntry.stringImmutablePropertyPrefixEntry(
+ LOCATION_PLACEHOLDER_PREFIX,
+ "The prefix of fileset placeholder property",
+ false /* required */,
+ null /* default value */,
+ false /* hidden */,
+ false /* reserved */))
.putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES)
.putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES)
.putAll(CredentialConfig.CREDENTIAL_PROPERTY_ENTRIES);
diff --git
a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java
b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java
index 6c438506a5..05c166df13 100644
---
a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java
+++
b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java
@@ -18,6 +18,7 @@
*/
package org.apache.gravitino.catalog.hadoop.integration.test;
+import static org.apache.gravitino.file.Fileset.LOCATION_PLACEHOLDER_PREFIX;
import static org.apache.gravitino.file.Fileset.Type.MANAGED;
import com.google.common.collect.ImmutableMap;
@@ -255,6 +256,32 @@ public class HadoopCatalogIT extends BaseIT {
Assertions.assertEquals(MANAGED, fileset3.type(), "fileset type should be
MANAGED by default");
}
+ @Test
+ public void testAlterFileset() {
+ // create fileset with placeholder in storage location
+ String filesetName = "test_alter_fileset";
+ String storageLocation = storageLocation(filesetName) + "/{{user}}";
+ String placeholderKey = LOCATION_PLACEHOLDER_PREFIX + "user";
+ createFileset(
+ filesetName, "comment", MANAGED, storageLocation,
ImmutableMap.of(placeholderKey, "test"));
+
+ // alter fileset
+ Exception exception =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ catalog
+ .asFilesetCatalog()
+ .alterFileset(
+ NameIdentifier.of(schemaName, filesetName),
+ FilesetChange.setProperty(placeholderKey, "test2")));
+ Assertions.assertTrue(
+ exception
+ .getMessage()
+ .contains("Property placeholder-user is immutable or reserved,
cannot be set"),
+ exception.getMessage());
+ }
+
@Test
public void testCreateFilesetWithChinese() throws IOException {
// create fileset
diff --git
a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalog.java
b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalog.java
index ddf7616318..910fe2db70 100644
---
a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalog.java
+++
b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalog.java
@@ -125,6 +125,8 @@ public class TestHiveCatalog extends
MiniHiveMetastoreService {
throwable
.getMessage()
.contains(
- String.format("Properties are required and must be set: [%s]",
METASTORE_URIS)));
+ String.format(
+ "Properties or property prefixes are required and must be
set: [%s]",
+ METASTORE_URIS)));
}
}
diff --git
a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java
b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java
index 52ebb8dab0..292325a435 100644
---
a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java
+++
b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java
@@ -218,7 +218,8 @@ public class CatalogKafkaIT extends BaseIT {
Assertions.assertTrue(
exception
.getMessage()
- .contains("Properties are required and must be set:
[bootstrap.servers]"));
+ .contains(
+ "Properties or property prefixes are required and must be set:
[bootstrap.servers]"));
exception =
Assertions.assertThrows(
@@ -233,7 +234,8 @@ public class CatalogKafkaIT extends BaseIT {
Assertions.assertTrue(
exception
.getMessage()
- .contains("Properties are required and must be set:
[bootstrap.servers]"));
+ .contains(
+ "Properties or property prefixes are required and must be set:
[bootstrap.servers]"));
// Test BOOTSTRAP_SERVERS that cannot be linked
String catalogName2 = GravitinoITUtils.genRandomName("test_catalog");
diff --git
a/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogHMSIT.java
b/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogHMSIT.java
index 9fc1c81b5f..7b2800f0f9 100644
---
a/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogHMSIT.java
+++
b/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogHMSIT.java
@@ -113,7 +113,13 @@ public class HudiCatalogHMSIT extends BaseIT {
Assertions.assertTrue(
exception
.getMessage()
- .contains("Properties are required and must be set:
[catalog-backend, uri]"),
+ .contains("Properties or property prefixes are required and must
be set"),
+ "Unexpected exception message: " + exception.getMessage());
+ Assertions.assertTrue(
+ exception.getMessage().contains("catalog-backend"),
+ "Unexpected exception message: " + exception.getMessage());
+ Assertions.assertTrue(
+ exception.getMessage().contains("uri"),
"Unexpected exception message: " + exception.getMessage());
// test testConnection exception
@@ -130,7 +136,13 @@ public class HudiCatalogHMSIT extends BaseIT {
Assertions.assertTrue(
exception
.getMessage()
- .contains("Properties are required and must be set:
[catalog-backend, uri]"),
+ .contains("Properties or property prefixes are required and must
be set"),
+ "Unexpected exception message: " + exception.getMessage());
+ Assertions.assertTrue(
+ exception.getMessage().contains("catalog-backend"),
+ "Unexpected exception message: " + exception.getMessage());
+ Assertions.assertTrue(
+ exception.getMessage().contains("uri"),
"Unexpected exception message: " + exception.getMessage());
// test testConnection
diff --git
a/core/src/main/java/org/apache/gravitino/catalog/PropertiesMetadataHelpers.java
b/core/src/main/java/org/apache/gravitino/catalog/PropertiesMetadataHelpers.java
index 54320c325c..c465b2a4cd 100644
---
a/core/src/main/java/org/apache/gravitino/catalog/PropertiesMetadataHelpers.java
+++
b/core/src/main/java/org/apache/gravitino/catalog/PropertiesMetadataHelpers.java
@@ -54,17 +54,23 @@ public class PropertiesMetadataHelpers {
.collect(Collectors.toList());
Preconditions.checkArgument(
reservedProperties.isEmpty(),
- "Properties are reserved and cannot be set: %s",
+ "Properties or property prefixes are reserved and cannot be set: %s",
reservedProperties);
List<String> absentProperties =
- propertiesMetadata.propertyEntries().keySet().stream()
- .filter(propertiesMetadata::isRequiredProperty)
- .filter(k -> !properties.containsKey(k))
+ propertiesMetadata.propertyEntries().values().stream()
+ .filter(PropertyEntry::isRequired)
+ .filter(
+ e ->
+ (!e.isPrefix() && !properties.containsKey(e.getName()))
+ || (e.isPrefix()
+ && properties.keySet().stream()
+ .noneMatch(k -> k.startsWith(e.getName()))))
+ .map(PropertyEntry::getName)
.collect(Collectors.toList());
Preconditions.checkArgument(
absentProperties.isEmpty(),
- "Properties are required and must be set: %s",
+ "Properties or property prefixes are required and must be set: %s",
absentProperties);
// use decode function to validate the property values
@@ -72,7 +78,7 @@ public class PropertiesMetadataHelpers {
String key = entry.getKey();
String value = entry.getValue();
if (propertiesMetadata.containsProperty(key)) {
- checkValueFormat(key, value,
propertiesMetadata.propertyEntries().get(key)::decode);
+ checkValueFormat(key, value,
propertiesMetadata.getPropertyEntry(key)::decode);
}
}
}
@@ -82,21 +88,29 @@ public class PropertiesMetadataHelpers {
Map<String, String> upserts,
Map<String, String> deletes) {
for (Map.Entry<String, String> entry : upserts.entrySet()) {
- PropertyEntry<?> propertyEntry =
propertiesMetadata.propertyEntries().get(entry.getKey());
+ if (!propertiesMetadata.containsProperty(entry.getKey())) {
+ continue;
+ }
+
+ PropertyEntry<?> propertyEntry =
propertiesMetadata.getPropertyEntry(entry.getKey());
if (Objects.nonNull(propertyEntry)) {
Preconditions.checkArgument(
!propertyEntry.isImmutable() && !propertyEntry.isReserved(),
- "Property " + propertyEntry.getName() + " is immutable or
reserved, cannot be set");
+ "Property " + entry.getKey() + " is immutable or reserved, cannot
be set");
checkValueFormat(entry.getKey(), entry.getValue(),
propertyEntry::decode);
}
}
for (Map.Entry<String, String> entry : deletes.entrySet()) {
- PropertyEntry<?> propertyEntry =
propertiesMetadata.propertyEntries().get(entry.getKey());
+ if (!propertiesMetadata.containsProperty(entry.getKey())) {
+ continue;
+ }
+
+ PropertyEntry<?> propertyEntry =
propertiesMetadata.getPropertyEntry(entry.getKey());
if (Objects.nonNull(propertyEntry)) {
Preconditions.checkArgument(
!propertyEntry.isImmutable() && !propertyEntry.isReserved(),
- "Property " + propertyEntry.getName() + " is immutable or
reserved, cannot be deleted");
+ "Property " + entry.getKey() + " is immutable or reserved, cannot
be deleted");
}
}
}
diff --git
a/core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java
b/core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java
index d4778b2ff9..46df3dd86f 100644
--- a/core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java
+++ b/core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java
@@ -18,7 +18,9 @@
*/
package org.apache.gravitino.connector;
+import java.util.Comparator;
import java.util.Map;
+import java.util.Optional;
import org.apache.gravitino.annotation.Evolving;
/** The PropertiesMetadata class is responsible for managing property
metadata. */
@@ -35,8 +37,13 @@ public interface PropertiesMetadata {
* @return true if the property is existed and reserved, false otherwise.
*/
default boolean isReservedProperty(String propertyName) {
- return propertyEntries().containsKey(propertyName)
- && propertyEntries().get(propertyName).isReserved();
+ // First check non-prefix exact match
+ if
(getNonPrefixEntry(propertyName).map(PropertyEntry::isReserved).orElse(false)) {
+ return true;
+ }
+
+ // Then check property prefixes
+ return
getPropertyPrefixEntry(propertyName).map(PropertyEntry::isReserved).orElse(false);
}
/**
@@ -46,8 +53,13 @@ public interface PropertiesMetadata {
* @return true if the property is existed and required, false otherwise.
*/
default boolean isRequiredProperty(String propertyName) {
- return propertyEntries().containsKey(propertyName)
- && propertyEntries().get(propertyName).isRequired();
+ // First check non-prefix exact match
+ if
(getNonPrefixEntry(propertyName).map(PropertyEntry::isRequired).orElse(false)) {
+ return true;
+ }
+
+ // Then check property prefixes
+ return
getPropertyPrefixEntry(propertyName).map(PropertyEntry::isRequired).orElse(false);
}
/**
@@ -57,8 +69,13 @@ public interface PropertiesMetadata {
* @return true if the property is existed and immutable, false otherwise.
*/
default boolean isImmutableProperty(String propertyName) {
- return propertyEntries().containsKey(propertyName)
- && propertyEntries().get(propertyName).isImmutable();
+ // First check non-prefix exact match
+ if
(getNonPrefixEntry(propertyName).map(PropertyEntry::isImmutable).orElse(false))
{
+ return true;
+ }
+
+ // Then check property prefixes
+ return
getPropertyPrefixEntry(propertyName).map(PropertyEntry::isImmutable).orElse(false);
}
/**
@@ -68,13 +85,19 @@ public interface PropertiesMetadata {
* @return true if the property is existed and hidden, false otherwise.
*/
default boolean isHiddenProperty(String propertyName) {
- return propertyEntries().containsKey(propertyName)
- && propertyEntries().get(propertyName).isHidden();
+ // First check non-prefix exact match
+ if
(getNonPrefixEntry(propertyName).map(PropertyEntry::isHidden).orElse(false)) {
+ return true;
+ }
+
+ // Then check property prefixes
+ return
getPropertyPrefixEntry(propertyName).map(PropertyEntry::isHidden).orElse(false);
}
/** @return true if the property is existed, false otherwise. */
default boolean containsProperty(String propertyName) {
- return propertyEntries().containsKey(propertyName);
+ return getNonPrefixEntry(propertyName).isPresent()
+ || getPropertyPrefixEntry(propertyName).isPresent();
}
/**
@@ -85,14 +108,12 @@ public interface PropertiesMetadata {
* @return the value object of the property.
*/
default Object getOrDefault(Map<String, String> properties, String
propertyName) {
- if (!containsProperty(propertyName)) {
- throw new IllegalArgumentException("Property is not defined: " +
propertyName);
- }
+ PropertyEntry<?> propertyEntry = getPropertyEntry(propertyName);
if (properties != null && properties.containsKey(propertyName)) {
- return
propertyEntries().get(propertyName).decode(properties.get(propertyName));
+ return propertyEntry.decode(properties.get(propertyName));
}
- return propertyEntries().get(propertyName).getDefaultValue();
+ return propertyEntry.getDefaultValue();
}
/**
@@ -102,10 +123,64 @@ public interface PropertiesMetadata {
* @return the default value object of the property.
*/
default Object getDefaultValue(String propertyName) {
+ PropertyEntry<?> propertyEntry = getPropertyEntry(propertyName);
+
+ return propertyEntry.getDefaultValue();
+ }
+
+ /**
+ * Get the property entry of the property. The non-prefix property entry
will be returned if
+ * exists, otherwise the longest prefix property entry will be returned.
+ *
+ * <p>For example, if there are two property prefix entries "foo." and
"foo.bar.", and the
+ * property name is "foo.bar.baz", the entry for "foo.bar." will be
returned. If the property
+ * entry "foo.bar.baz" is defined, it will be returned instead.
+ *
+ * @param propertyName The name of the property.
+ * @return the property entry object of the property.
+ * @throws IllegalArgumentException if the property is not defined.
+ */
+ default PropertyEntry<?> getPropertyEntry(String propertyName) throws
IllegalArgumentException {
if (!containsProperty(propertyName)) {
throw new IllegalArgumentException("Property is not defined: " +
propertyName);
}
+ return getNonPrefixEntry(propertyName).isPresent()
+ ? getNonPrefixEntry(propertyName).get()
+ : getPropertyPrefixEntry(propertyName).get();
+ }
+
+ /**
+ * Get the property prefix entry of the property. If there are multiple
property prefix entries
+ * matching the property name, the longest prefix entry will be returned.
+ *
+ * <p>For example, if there are two property prefix entries "foo." and
"foo.bar.", and the
+ * property name is "foo.bar.baz", the entry for "foo.bar." will be returned.
+ *
+ * @param propertyName The name of the property.
+ * @return an Optional containing the property prefix entry if it exists, or
empty Optional
+ * otherwise.
+ */
+ default Optional<PropertyEntry<?>> getPropertyPrefixEntry(String
propertyName) {
+ return propertyEntries().entrySet().stream()
+ .filter(e -> e.getValue().isPrefix() &&
propertyName.startsWith(e.getKey()))
+ // get the longest prefix property
+ .max(Map.Entry.comparingByKey(Comparator.comparingInt(String::length)))
+ .map(Map.Entry::getValue);
+ }
- return propertyEntries().get(propertyName).getDefaultValue();
+ /**
+ * Get the non-prefix property entry of the property. That is, the property
entry that is not a
+ * prefix.
+ *
+ * @param propertyName The name of the property.
+ * @return an Optional containing the non-prefix property entry if it
exists, or empty Optional
+ * otherwise.
+ */
+ default Optional<PropertyEntry<?>> getNonPrefixEntry(String propertyName) {
+ if (propertyEntries().containsKey(propertyName)
+ && !propertyEntries().get(propertyName).isPrefix()) {
+ return Optional.of(propertyEntries().get(propertyName));
+ }
+ return Optional.empty();
}
}
diff --git
a/core/src/main/java/org/apache/gravitino/connector/PropertyEntry.java
b/core/src/main/java/org/apache/gravitino/connector/PropertyEntry.java
index a32c2fff21..fff77ae677 100644
--- a/core/src/main/java/org/apache/gravitino/connector/PropertyEntry.java
+++ b/core/src/main/java/org/apache/gravitino/connector/PropertyEntry.java
@@ -40,6 +40,7 @@ public final class PropertyEntry<T> {
private final Function<T, String> encoder;
private final boolean hidden;
private final boolean reserved;
+ private final boolean prefix;
/**
* @param name The name of the property
@@ -66,6 +67,46 @@ public final class PropertyEntry<T> {
Function<T, String> encoder,
boolean hidden,
boolean reserved) {
+ this(
+ name,
+ description,
+ required,
+ immutable,
+ javaType,
+ defaultValue,
+ decoder,
+ encoder,
+ hidden,
+ reserved,
+ false);
+ }
+
+ /**
+ * @param name The name of the property
+ * @param description Describe the purpose of this property
+ * @param required Whether this property is required. If true, the property
must be set when
+ * creating a table
+ * @param immutable Whether this property is immutable. If true, the
property cannot be changed by
+ * user after the table is created
+ * @param javaType The java type of the property
+ * @param defaultValue Non-required property can have a default value
+ * @param decoder Decode the string value to the java type
+ * @param encoder Encode the java type to the string value
+ * @param hidden Whether this property is hidden from user, such as password
+ * @param reserved This property is reserved and cannot be set by user
+ */
+ private PropertyEntry(
+ String name,
+ String description,
+ boolean required,
+ boolean immutable,
+ Class<T> javaType,
+ T defaultValue,
+ Function<String, T> decoder,
+ Function<T, String> encoder,
+ boolean hidden,
+ boolean reserved,
+ boolean prefix) {
Preconditions.checkArgument(StringUtils.isNotBlank(name), "name cannot be
null or empty");
Preconditions.checkArgument(
StringUtils.isNotBlank(description), "description cannot be null or
empty");
@@ -88,6 +129,7 @@ public final class PropertyEntry<T> {
this.encoder = encoder;
this.hidden = hidden;
this.reserved = reserved;
+ this.prefix = prefix;
}
public static class Builder<T> {
@@ -102,6 +144,7 @@ public final class PropertyEntry<T> {
private Function<T, String> encoder;
private boolean hidden;
private boolean reserved;
+ private boolean prefix;
public Builder<T> withName(String name) {
this.name = name;
@@ -153,8 +196,13 @@ public final class PropertyEntry<T> {
return this;
}
+ public Builder<T> withPrefix(boolean prefix) {
+ this.prefix = prefix;
+ return this;
+ }
+
public PropertyEntry<T> build() {
- return new PropertyEntry<T>(
+ return new PropertyEntry<>(
name,
description,
required,
@@ -164,7 +212,8 @@ public final class PropertyEntry<T> {
decoder,
encoder,
hidden,
- reserved);
+ reserved,
+ prefix);
}
}
@@ -172,6 +221,29 @@ public final class PropertyEntry<T> {
return decoder.apply(value);
}
+ public static PropertyEntry<String> stringPropertyPrefixEntry(
+ String name,
+ String description,
+ boolean required,
+ boolean immutable,
+ String defaultValue,
+ boolean hidden,
+ boolean reserved) {
+ return new Builder<String>()
+ .withName(name)
+ .withDescription(description)
+ .withRequired(required)
+ .withImmutable(immutable)
+ .withJavaType(String.class)
+ .withDefaultValue(defaultValue)
+ .withDecoder(Function.identity())
+ .withEncoder(Function.identity())
+ .withHidden(hidden)
+ .withReserved(reserved)
+ .withPrefix(true)
+ .build();
+ }
+
public static PropertyEntry<String> stringPropertyEntry(
String name,
String description,
@@ -354,6 +426,39 @@ public final class PropertyEntry<T> {
return stringPropertyEntry(name, description, required, true,
defaultValue, hidden, reserved);
}
+ public static PropertyEntry<String> stringImmutablePropertyPrefixEntry(
+ String name,
+ String description,
+ boolean required,
+ String defaultValue,
+ boolean hidden,
+ boolean reserved) {
+ return stringPropertyPrefixEntry(
+ name, description, required, true /* immutable */, defaultValue,
hidden, reserved);
+ }
+
+ public static PropertyEntry<String> stringRequiredPropertyPrefixEntry(
+ String name,
+ String description,
+ boolean immutable,
+ String defaultValue,
+ boolean hidden,
+ boolean reserved) {
+ return stringPropertyPrefixEntry(
+ name, description, true /* required */, immutable, defaultValue,
hidden, reserved);
+ }
+
+ public static PropertyEntry<String> stringOptionalPropertyPrefixEntry(
+ String name,
+ String description,
+ boolean immutable,
+ String defaultValue,
+ boolean hidden,
+ boolean reserved) {
+ return stringPropertyPrefixEntry(
+ name, description, false /* required */, immutable, defaultValue,
hidden, reserved);
+ }
+
public static <T extends Enum<T>> PropertyEntry<T> enumPropertyEntry(
String name,
String description,
diff --git a/core/src/test/java/org/apache/gravitino/TestCatalog.java
b/core/src/test/java/org/apache/gravitino/TestCatalog.java
index 420396559d..fe353b94e9 100644
--- a/core/src/test/java/org/apache/gravitino/TestCatalog.java
+++ b/core/src/test/java/org/apache/gravitino/TestCatalog.java
@@ -39,6 +39,15 @@ public class TestCatalog extends BaseCatalog<TestCatalog> {
private static final TestFilesetPropertiesMetadata
FILESET_PROPERTIES_METADATA =
new TestFilesetPropertiesMetadata();
+ public static final String PROPERTY_KEY1 = "key1";
+ public static final String PROPERTY_KEY2 = "key2";
+ public static final String PROPERTY_KEY3 = "key3";
+ public static final String PROPERTY_KEY4 = "key4";
+ public static final String PROPERTY_RESERVED_KEY = "reserved_key";
+ public static final String PROPERTY_HIDDEN_KEY = "hidden_key";
+ public static final String PROPERTY_KEY5_PREFIX = "key5-";
+ public static final String PROPERTY_KEY6_PREFIX = "key6-";
+
public TestCatalog() {}
@Override
@@ -73,14 +82,27 @@ public class TestCatalog extends BaseCatalog<TestCatalog> {
protected Map<String, PropertyEntry<?>> specificPropertyEntries() {
return ImmutableMap.<String, PropertyEntry<?>>builder()
.put(
- "key1",
- PropertyEntry.stringPropertyEntry("key1", "value1", true,
true, null, false, false))
+ PROPERTY_KEY1,
+ PropertyEntry.stringPropertyEntry(
+ PROPERTY_KEY1,
+ "value1" /* description */,
+ true /* required */,
+ true /* immutable */,
+ null /* default value*/,
+ false /* hidden */,
+ false /* reserved */))
.put(
- "key2",
+ PROPERTY_KEY2,
PropertyEntry.stringPropertyEntry(
- "key2", "value2", true, false, null, false, false))
+ PROPERTY_KEY2,
+ "value2" /* description */,
+ true /* required */,
+ false /* immutable */,
+ null /* default value*/,
+ false /* hidden */,
+ false /* reserved */))
.put(
- "key3",
+ PROPERTY_KEY3,
new PropertyEntry.Builder<Integer>()
.withDecoder(Integer::parseInt)
.withEncoder(Object::toString)
@@ -91,30 +113,60 @@ public class TestCatalog extends BaseCatalog<TestCatalog> {
.withImmutable(true)
.withJavaType(Integer.class)
.withRequired(false)
- .withName("key3")
+ .withName(PROPERTY_KEY3)
.build())
.put(
- "key4",
+ PROPERTY_KEY4,
PropertyEntry.stringPropertyEntry(
- "key4", "value4", false, false, "value4", false, false))
+ PROPERTY_KEY4, "value4", false, false, "value4", false,
false))
.put(
- "reserved_key",
+ PROPERTY_RESERVED_KEY,
PropertyEntry.stringPropertyEntry(
- "reserved_key", "reserved_key", false, true,
"reserved_value", false, true))
+ PROPERTY_RESERVED_KEY,
+ "reserved_key" /* description */,
+ false /* required */,
+ true /* immutable */,
+ "reserved_value" /* default value*/,
+ false /* hidden */,
+ true /* reserved */))
.put(
- "hidden_key",
+ PROPERTY_HIDDEN_KEY,
PropertyEntry.stringPropertyEntry(
- "hidden_key", "hidden_key", false, false, "hidden_value",
true, false))
+ PROPERTY_HIDDEN_KEY,
+ "hidden_key" /* description */,
+ false /* required */,
+ false /* immutable */,
+ "hidden_value" /* default value*/,
+ true /* hidden */,
+ false /* reserved */))
.put(
FAIL_CREATE,
PropertyEntry.booleanPropertyEntry(
FAIL_CREATE,
"Whether an exception needs to be thrown on creation",
- false,
- false,
- false,
- false,
- false))
+ false /* required */,
+ false /* immutable */,
+ false /* default value*/,
+ false /* hidden */,
+ false /* reserved */))
+ .put(
+ PROPERTY_KEY5_PREFIX,
+ PropertyEntry.stringRequiredPropertyPrefixEntry(
+ PROPERTY_KEY5_PREFIX,
+ "property with prefix 'key5-'",
+ false /* immutable */,
+ null /* default value*/,
+ false /* hidden */,
+ false /* reserved */))
+ .put(
+ PROPERTY_KEY6_PREFIX,
+ PropertyEntry.stringImmutablePropertyPrefixEntry(
+ PROPERTY_KEY6_PREFIX,
+ "property with prefix 'key6-'",
+ false /* required */,
+ null /* default value*/,
+ false /* hidden */,
+ false /* reserved */))
.build();
}
};
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestCatalogManager.java
b/core/src/test/java/org/apache/gravitino/catalog/TestCatalogManager.java
index af4dee8654..acb8886a52 100644
--- a/core/src/test/java/org/apache/gravitino/catalog/TestCatalogManager.java
+++ b/core/src/test/java/org/apache/gravitino/catalog/TestCatalogManager.java
@@ -19,6 +19,12 @@
package org.apache.gravitino.catalog;
import static org.apache.gravitino.StringIdentifier.ID_KEY;
+import static org.apache.gravitino.TestCatalog.PROPERTY_KEY1;
+import static org.apache.gravitino.TestCatalog.PROPERTY_KEY2;
+import static org.apache.gravitino.TestCatalog.PROPERTY_KEY3;
+import static org.apache.gravitino.TestCatalog.PROPERTY_KEY4;
+import static org.apache.gravitino.TestCatalog.PROPERTY_KEY5_PREFIX;
+import static org.apache.gravitino.TestCatalog.PROPERTY_KEY6_PREFIX;
import static org.mockito.ArgumentMatchers.any;
import com.google.common.collect.ImmutableMap;
@@ -28,17 +34,20 @@ import java.io.IOException;
import java.time.Instant;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.gravitino.Catalog;
import org.apache.gravitino.CatalogChange;
import org.apache.gravitino.Config;
import org.apache.gravitino.Configs;
import org.apache.gravitino.EntityStore;
+import org.apache.gravitino.GravitinoEnv;
import org.apache.gravitino.NameIdentifier;
import org.apache.gravitino.Namespace;
import org.apache.gravitino.exceptions.CatalogAlreadyExistsException;
import org.apache.gravitino.exceptions.CatalogInUseException;
import org.apache.gravitino.exceptions.NoSuchCatalogException;
import org.apache.gravitino.exceptions.NoSuchMetalakeException;
+import org.apache.gravitino.lock.LockManager;
import org.apache.gravitino.meta.AuditInfo;
import org.apache.gravitino.meta.BaseMetalake;
import org.apache.gravitino.meta.SchemaEntity;
@@ -77,7 +86,7 @@ public class TestCatalogManager {
.build();
@BeforeAll
- public static void setUp() throws IOException {
+ public static void setUp() throws IOException, IllegalAccessException {
config = new Config(false) {};
config.set(Configs.CATALOG_LOAD_ISOLATED, false);
@@ -87,6 +96,7 @@ public class TestCatalogManager {
entityStore.put(metalakeEntity, true);
catalogManager = new CatalogManager(config, entityStore, new
RandomIdGenerator());
+ FieldUtils.writeField(GravitinoEnv.getInstance(), "lockManager", new
LockManager(config), true);
catalogManager = Mockito.spy(catalogManager);
}
@@ -123,8 +133,9 @@ public class TestCatalogManager {
// key1 is required;
Map<String, String> props1 =
ImmutableMap.<String, String>builder()
- .put("key2", "value2")
- .put("key1", "value1")
+ .put(PROPERTY_KEY2, "value2")
+ .put(PROPERTY_KEY1, "value1")
+ .put(PROPERTY_KEY5_PREFIX + "1", "value1")
.put("mock", "mock")
.build();
Assertions.assertDoesNotThrow(
@@ -136,10 +147,12 @@ public class TestCatalogManager {
// key1 is required;
Map<String, String> props2 =
ImmutableMap.<String, String>builder()
- .put("key2", "value2")
- .put("key1", "value1")
- .put("key3", "3")
- .put("key4", "value4")
+ .put(PROPERTY_KEY2, "value2")
+ .put(PROPERTY_KEY1, "value1")
+ .put(PROPERTY_KEY3, "3")
+ .put(PROPERTY_KEY4, "value4")
+ .put(PROPERTY_KEY5_PREFIX + "1", "value1")
+ .put(PROPERTY_KEY6_PREFIX + "1", "value1")
.put("mock", "mock")
.build();
Assertions.assertDoesNotThrow(
@@ -147,30 +160,41 @@ public class TestCatalogManager {
catalogManager.createCatalog(
ident2, Catalog.Type.RELATIONAL, provider, "comment", props2));
- CatalogChange change1 = CatalogChange.setProperty("key1", "value1");
+ CatalogChange change1 = CatalogChange.setProperty(PROPERTY_KEY1, "value1");
Exception e1 =
Assertions.assertThrows(
IllegalArgumentException.class, () ->
catalogManager.alterCatalog(ident, change1));
Assertions.assertTrue(e1.getMessage().contains("Property key1 is
immutable"));
- CatalogChange change2 = CatalogChange.setProperty("key3", "value2");
+ CatalogChange change2 = CatalogChange.setProperty(PROPERTY_KEY3, "value2");
Exception e2 =
Assertions.assertThrows(
IllegalArgumentException.class, () ->
catalogManager.alterCatalog(ident2, change2));
Assertions.assertTrue(e2.getMessage().contains("Property key3 is
immutable"));
Assertions.assertDoesNotThrow(
- () -> catalogManager.alterCatalog(ident2,
CatalogChange.setProperty("key4", "value4")));
+ () ->
+ catalogManager.alterCatalog(
+ ident2, CatalogChange.setProperty(PROPERTY_KEY4, "value4")));
Assertions.assertDoesNotThrow(
- () -> catalogManager.alterCatalog(ident2,
CatalogChange.setProperty("key2", "value2")));
+ () ->
+ catalogManager.alterCatalog(
+ ident2, CatalogChange.setProperty(PROPERTY_KEY2, "value2")));
- CatalogChange change3 = CatalogChange.setProperty("key4", "value4");
- CatalogChange change4 = CatalogChange.removeProperty("key1");
+ CatalogChange change3 = CatalogChange.setProperty(PROPERTY_KEY4, "value4");
+ CatalogChange change4 = CatalogChange.removeProperty(PROPERTY_KEY1);
Exception e3 =
Assertions.assertThrows(
IllegalArgumentException.class,
() -> catalogManager.alterCatalog(ident2, change3, change4));
Assertions.assertTrue(e3.getMessage().contains("Property key1 is
immutable"));
+
+ CatalogChange change5 = CatalogChange.setProperty(PROPERTY_KEY6_PREFIX +
"1", "value1");
+ e3 =
+ Assertions.assertThrows(
+ IllegalArgumentException.class, () ->
catalogManager.alterCatalog(ident2, change5));
+ Assertions.assertTrue(
+ e3.getMessage().contains("Property key6-1 is immutable"),
e3.getMessage());
reset();
}
@@ -186,37 +210,62 @@ public class TestCatalogManager {
// key1 is required;
Map<String, String> props1 =
- ImmutableMap.<String, String>builder().put("key2",
"value2").put("mock", "mock").build();
+ ImmutableMap.<String, String>builder()
+ .put(PROPERTY_KEY2, "value2")
+ .put(PROPERTY_KEY5_PREFIX + "1", "value1")
+ .put("mock", "mock")
+ .build();
IllegalArgumentException e1 =
Assertions.assertThrows(
IllegalArgumentException.class,
() ->
catalogManager.createCatalog(
ident, Catalog.Type.RELATIONAL, provider, "comment",
props1));
- Assertions.assertTrue(
- e1.getMessage().contains("Properties are required and must be set:
[key1]"));
+ Assertions.assertEquals(
+ "Properties or property prefixes are required and must be set:
[key1]", e1.getMessage());
// BUG here, in memory does not support rollback
reset();
// key2 is required;
Map<String, String> props2 =
- ImmutableMap.<String, String>builder().put("key1",
"value1").put("mock", "mock").build();
+ ImmutableMap.<String, String>builder()
+ .put(PROPERTY_KEY1, "value1")
+ .put(PROPERTY_KEY5_PREFIX + "1", "value2")
+ .put("mock", "mock")
+ .build();
e1 =
Assertions.assertThrows(
IllegalArgumentException.class,
() ->
catalogManager.createCatalog(
ident, Catalog.Type.RELATIONAL, provider, "comment",
props2));
- Assertions.assertTrue(
- e1.getMessage().contains("Properties are required and must be set:
[key2]"));
+ Assertions.assertEquals(
+ "Properties or property prefixes are required and must be set:
[key2]", e1.getMessage());
reset();
+ // property with fixed prefix key5- is required;
+ Map<String, String> props4 =
+ ImmutableMap.<String, String>builder()
+ .put(PROPERTY_KEY1, "value1")
+ .put(PROPERTY_KEY2, "value2")
+ .put("mock", "mock")
+ .build();
+ e1 =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ catalogManager.createCatalog(
+ ident, Catalog.Type.RELATIONAL, provider, "comment",
props4));
+ Assertions.assertEquals(
+ "Properties or property prefixes are required and must be set:
[key5-]", e1.getMessage());
+
// key3 is optional, but we assign a wrong value format
Map<String, String> props3 =
ImmutableMap.<String, String>builder()
- .put("key1", "value1")
- .put("key2", "value2")
- .put("key3", "a12a1a")
+ .put(PROPERTY_KEY1, "value1")
+ .put(PROPERTY_KEY2, "value2")
+ .put(PROPERTY_KEY3, "a12a1a")
+ .put(PROPERTY_KEY5_PREFIX + "1", "value1")
.put("mock", "mock")
.build();
e1 =
@@ -225,7 +274,7 @@ public class TestCatalogManager {
() ->
catalogManager.createCatalog(
ident, Catalog.Type.RELATIONAL, provider, "comment",
props3));
- Assertions.assertTrue(e1.getMessage().contains("Invalid value: 'a12a1a'
for property: 'key3'"));
+ Assertions.assertEquals("Invalid value: 'a12a1a' for property: 'key3'",
e1.getMessage());
reset();
}
@@ -234,8 +283,9 @@ public class TestCatalogManager {
NameIdentifier ident = NameIdentifier.of("metalake", "test1");
Map<String, String> props = Maps.newHashMap();
- props.put("key1", "value1");
- props.put("key2", "value2");
+ props.put(PROPERTY_KEY1, "value1");
+ props.put(PROPERTY_KEY2, "value2");
+ props.put(PROPERTY_KEY5_PREFIX + "1", "value3");
// test before creation
Assertions.assertDoesNotThrow(
@@ -301,7 +351,9 @@ public class TestCatalogManager {
catalogManager.createCatalog(
failedIdent, Catalog.Type.RELATIONAL, provider, "comment",
props));
Assertions.assertTrue(
- exception3.getMessage().contains("Properties are reserved and cannot
be set"),
+ exception3
+ .getMessage()
+ .contains("Properties or property prefixes are reserved and cannot
be set"),
exception3.getMessage());
Assertions.assertNull(CatalogManager.catalogCache.getIfPresent(failedIdent));
// Test failed for the second time
@@ -312,7 +364,9 @@ public class TestCatalogManager {
catalogManager.createCatalog(
failedIdent, Catalog.Type.RELATIONAL, provider, "comment",
props));
Assertions.assertTrue(
- exception4.getMessage().contains("Properties are reserved and cannot
be set"),
+ exception4
+ .getMessage()
+ .contains("Properties or property prefixes are reserved and cannot
be set"),
exception4.getMessage());
Assertions.assertNull(CatalogManager.catalogCache.getIfPresent(failedIdent));
}
@@ -322,7 +376,15 @@ public class TestCatalogManager {
NameIdentifier ident = NameIdentifier.of("metalake", "test11");
NameIdentifier ident1 = NameIdentifier.of("metalake", "test12");
Map<String, String> props =
- ImmutableMap.of("provider", "test", "key1", "value1", "key2",
"value2");
+ ImmutableMap.of(
+ "provider",
+ "test",
+ PROPERTY_KEY1,
+ "value1",
+ PROPERTY_KEY2,
+ "value2",
+ PROPERTY_KEY5_PREFIX + "1",
+ "value3");
catalogManager.createCatalog(ident, Catalog.Type.RELATIONAL, provider,
"comment", props);
catalogManager.createCatalog(ident1, Catalog.Type.RELATIONAL, provider,
"comment", props);
@@ -345,7 +407,15 @@ public class TestCatalogManager {
NameIdentifier relIdent = NameIdentifier.of("metalake", "catalog_rel");
NameIdentifier fileIdent = NameIdentifier.of("metalake", "catalog_file");
Map<String, String> props =
- ImmutableMap.of("provider", "test", "key1", "value1", "key2",
"value2");
+ ImmutableMap.of(
+ "provider",
+ "test",
+ PROPERTY_KEY1,
+ "value1",
+ PROPERTY_KEY2,
+ "value2",
+ PROPERTY_KEY5_PREFIX + "1",
+ "value3");
catalogManager.createCatalog(relIdent, Catalog.Type.RELATIONAL, provider,
"comment", props);
catalogManager.createCatalog(fileIdent, Catalog.Type.FILESET, provider,
"comment", props);
@@ -378,7 +448,15 @@ public class TestCatalogManager {
public void testLoadCatalog() {
NameIdentifier ident = NameIdentifier.of("metalake", "test21");
Map<String, String> props =
- ImmutableMap.of("provider", "test", "key1", "value1", "key2",
"value2");
+ ImmutableMap.of(
+ "provider",
+ "test",
+ PROPERTY_KEY1,
+ "value1",
+ PROPERTY_KEY2,
+ "value2",
+ PROPERTY_KEY5_PREFIX + "1",
+ "value3");
catalogManager.createCatalog(ident, Catalog.Type.RELATIONAL, provider,
"comment", props);
@@ -404,7 +482,15 @@ public class TestCatalogManager {
public void testAlterCatalog() {
NameIdentifier ident = NameIdentifier.of("metalake", "test31");
Map<String, String> props =
- ImmutableMap.of("provider", "test", "key1", "value1", "key2",
"value2");
+ ImmutableMap.of(
+ "provider",
+ "test",
+ PROPERTY_KEY1,
+ "value1",
+ PROPERTY_KEY2,
+ "value2",
+ PROPERTY_KEY5_PREFIX + "1",
+ "value3");
String comment = "comment";
catalogManager.createCatalog(ident, Catalog.Type.RELATIONAL, provider,
comment, props);
@@ -451,7 +537,15 @@ public class TestCatalogManager {
public void testDropCatalog() {
NameIdentifier ident = NameIdentifier.of("metalake", "test41");
Map<String, String> props =
- ImmutableMap.of("provider", "test", "key1", "value1", "key2",
"value2");
+ ImmutableMap.of(
+ "provider",
+ "test",
+ PROPERTY_KEY1,
+ "value1",
+ PROPERTY_KEY2,
+ "value2",
+ PROPERTY_KEY5_PREFIX + "1",
+ "value3");
String comment = "comment";
catalogManager.createCatalog(ident, Catalog.Type.RELATIONAL, provider,
comment, props);
@@ -479,7 +573,15 @@ public class TestCatalogManager {
public void testForceDropCatalog() throws Exception {
NameIdentifier ident = NameIdentifier.of("metalake", "test41");
Map<String, String> props =
- ImmutableMap.of("provider", "test", "key1", "value1", "key2",
"value2");
+ ImmutableMap.of(
+ "provider",
+ "test",
+ PROPERTY_KEY1,
+ "value1",
+ PROPERTY_KEY2,
+ "value2",
+ PROPERTY_KEY5_PREFIX + "1",
+ "value3");
String comment = "comment";
catalogManager.createCatalog(ident, Catalog.Type.RELATIONAL, provider,
comment, props);
SchemaEntity schemaEntity =
@@ -507,7 +609,15 @@ public class TestCatalogManager {
void testAlterMutableProperties() {
NameIdentifier ident = NameIdentifier.of("metalake", "test41");
Map<String, String> props =
- ImmutableMap.of("provider", "test", "key1", "value1", "key2",
"value2");
+ ImmutableMap.of(
+ "provider",
+ "test",
+ PROPERTY_KEY1,
+ "value1",
+ PROPERTY_KEY2,
+ "value2",
+ PROPERTY_KEY5_PREFIX + "1",
+ "value3");
String comment = "comment";
Catalog oldCatalog =
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestCatalogNormalizeDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestCatalogNormalizeDispatcher.java
index 66cf9874ca..7904f7494f 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestCatalogNormalizeDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestCatalogNormalizeDispatcher.java
@@ -19,6 +19,9 @@
package org.apache.gravitino.catalog;
import static org.apache.gravitino.Catalog.Type.RELATIONAL;
+import static org.apache.gravitino.TestCatalog.PROPERTY_KEY1;
+import static org.apache.gravitino.TestCatalog.PROPERTY_KEY2;
+import static org.apache.gravitino.TestCatalog.PROPERTY_KEY5_PREFIX;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
@@ -98,7 +101,14 @@ public class TestCatalogNormalizeDispatcher {
String[] legalNames = {"catalog", "_catalog", "1_catalog", "_", "1"};
for (String legalName : legalNames) {
NameIdentifier catalogIdent = NameIdentifier.of(metalake, legalName);
- Map<String, String> props = ImmutableMap.of("key1", "value1", "key2",
"value2");
+ Map<String, String> props =
+ ImmutableMap.of(
+ PROPERTY_KEY1,
+ "value1",
+ PROPERTY_KEY2,
+ "value2",
+ PROPERTY_KEY5_PREFIX + "1",
+ "value3");
Catalog catalog =
catalogNormalizeDispatcher.createCatalog(catalogIdent, RELATIONAL,
"test", null, props);
Assertions.assertEquals(legalName, catalog.name());
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java
index b9b80b18c0..9dd411b28b 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java
@@ -83,14 +83,14 @@ public class TestFilesetOperationDispatcher extends
TestOperationDispatcher {
() ->
filesetOperationDispatcher.createFileset(
filesetIdent1, "comment", Fileset.Type.MANAGED, "test",
illegalProps),
- "Properties are required and must be set");
+ "Properties or property prefixes are required and must be set");
Map<String, String> illegalProps2 = ImmutableMap.of("k1", "v1", ID_KEY,
"test");
testPropertyException(
() ->
filesetOperationDispatcher.createFileset(
filesetIdent1, "comment", Fileset.Type.MANAGED, "test",
illegalProps2),
- "Properties are reserved and cannot be set",
+ "Properties or property prefixes are reserved and cannot be set",
"gravitino.identifier");
}
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestModelOperationDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestModelOperationDispatcher.java
index b7aada3eef..dd1a1d48c3 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestModelOperationDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestModelOperationDispatcher.java
@@ -94,7 +94,7 @@ public class TestModelOperationDispatcher extends
TestOperationDispatcher {
Map<String, String> illegalProps = ImmutableMap.of("k1", "v1", ID_KEY,
"test");
testPropertyException(
() -> modelOperationDispatcher.registerModel(modelIdent, "comment",
illegalProps),
- "Properties are reserved and cannot be set",
+ "Properties or property prefixes are reserved and cannot be set",
ID_KEY);
}
@@ -190,7 +190,7 @@ public class TestModelOperationDispatcher extends
TestOperationDispatcher {
() ->
modelOperationDispatcher.linkModelVersion(
modelIdent, "path", aliases, "comment", illegalProps),
- "Properties are reserved and cannot be set",
+ "Properties or property prefixes are reserved and cannot be set",
ID_KEY);
}
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestOperationDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestOperationDispatcher.java
index 99a51e10d4..5b83a0e22b 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestOperationDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestOperationDispatcher.java
@@ -21,6 +21,9 @@ package org.apache.gravitino.catalog;
import static org.apache.gravitino.Configs.TREE_LOCK_CLEAN_INTERVAL;
import static org.apache.gravitino.Configs.TREE_LOCK_MAX_NODE_IN_MEMORY;
import static org.apache.gravitino.Configs.TREE_LOCK_MIN_NODE_IN_MEMORY;
+import static org.apache.gravitino.TestCatalog.PROPERTY_KEY1;
+import static org.apache.gravitino.TestCatalog.PROPERTY_KEY2;
+import static org.apache.gravitino.TestCatalog.PROPERTY_KEY5_PREFIX;
import static
org.apache.gravitino.TestFilesetPropertiesMetadata.TEST_FILESET_HIDDEN_KEY;
import static
org.apache.gravitino.utils.NameIdentifierUtil.getCatalogIdentifier;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -103,7 +106,9 @@ public abstract class TestOperationDispatcher {
FieldUtils.writeField(GravitinoEnv.getInstance(), "lockManager", new
LockManager(config), true);
NameIdentifier ident = NameIdentifier.of(metalake, catalog);
- Map<String, String> props = ImmutableMap.of("key1", "value1", "key2",
"value2");
+ Map<String, String> props =
+ ImmutableMap.of(
+ PROPERTY_KEY1, "value1", PROPERTY_KEY2, "value2",
PROPERTY_KEY5_PREFIX + "1", "value3");
catalogManager.createCatalog(ident, Catalog.Type.RELATIONAL, "test",
"comment", props);
}
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestSchemaOperationDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestSchemaOperationDispatcher.java
index 5f7ef050e2..f57a8c4038 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestSchemaOperationDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestSchemaOperationDispatcher.java
@@ -92,14 +92,14 @@ public class TestSchemaOperationDispatcher extends
TestOperationDispatcher {
testPropertyException(
() -> dispatcher.createSchema(schemaIdent, "comment",
illegalSchemaProperties),
- "Properties are required and must be set");
+ "Properties or property prefixes are required and must be set");
// Test reserved table properties exception
illegalSchemaProperties.put(COMMENT_KEY, "table comment");
illegalSchemaProperties.put(ID_KEY, "gravitino.v1.uidfdsafdsa");
testPropertyException(
() -> dispatcher.createSchema(schemaIdent, "comment",
illegalSchemaProperties),
- "Properties are reserved and cannot be set",
+ "Properties or property prefixes are reserved and cannot be set",
"comment",
"gravitino.identifier");
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java
index cbdbc4848a..d0016776c9 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java
@@ -130,7 +130,7 @@ public class TestTableOperationDispatcher extends
TestOperationDispatcher {
() ->
tableOperationDispatcher.createTable(
tableIdent1, columns, "comment", illegalTableProperties, new
Transform[0]),
- "Properties are required and must be set");
+ "Properties or property prefixes are required and must be set");
// Test reserved table properties exception
illegalTableProperties.put(COMMENT_KEY, "table comment");
@@ -139,7 +139,7 @@ public class TestTableOperationDispatcher extends
TestOperationDispatcher {
() ->
tableOperationDispatcher.createTable(
tableIdent1, columns, "comment", illegalTableProperties, new
Transform[0]),
- "Properties are reserved and cannot be set",
+ "Properties or property prefixes are reserved and cannot be set",
"comment",
"gravitino.identifier");
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestTopicOperationDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestTopicOperationDispatcher.java
index 7ee545e8e5..0aea8c6604 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestTopicOperationDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestTopicOperationDispatcher.java
@@ -24,6 +24,8 @@ import static
org.apache.gravitino.Configs.TREE_LOCK_MIN_NODE_IN_MEMORY;
import static org.apache.gravitino.Entity.EntityType.SCHEMA;
import static org.apache.gravitino.StringIdentifier.ID_KEY;
import static org.apache.gravitino.TestBasePropertiesMetadata.COMMENT_KEY;
+import static org.apache.gravitino.TestCatalog.PROPERTY_KEY1;
+import static org.apache.gravitino.TestCatalog.PROPERTY_KEY5_PREFIX;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.eq;
@@ -96,12 +98,13 @@ public class TestTopicOperationDispatcher extends
TestOperationDispatcher {
Map<String, String> illegalProps = ImmutableMap.of("k2", "v2");
testPropertyException(
() -> topicOperationDispatcher.createTopic(topicIdent1, "comment",
null, illegalProps),
- "Properties are required and must be set");
+ "Properties or property prefixes are required and must be set");
- Map<String, String> illegalProps2 = ImmutableMap.of("k1", "v1", ID_KEY,
"test");
+ Map<String, String> illegalProps2 =
+ ImmutableMap.of(PROPERTY_KEY1, "v1", PROPERTY_KEY5_PREFIX + "1",
"value1", ID_KEY, "test");
testPropertyException(
() -> topicOperationDispatcher.createTopic(topicIdent1, "comment",
null, illegalProps2),
- "Properties are reserved and cannot be set",
+ "Properties or property prefixes are reserved and cannot be set",
"gravitino.identifier");
}
diff --git a/docs/hadoop-catalog.md b/docs/hadoop-catalog.md
index 3b382235f8..1c255484e5 100644
--- a/docs/hadoop-catalog.md
+++ b/docs/hadoop-catalog.md
@@ -122,22 +122,22 @@ Refer to [Schema
operation](./manage-fileset-metadata-using-gravitino.md#schema-
### Fileset properties
-| Property name | Description
| Default
value | Required | Since Version |
-|---------------------------------------|--------------------------------------------------------------------------------------------------------|--------------------------|----------|------------------|
-| `authentication.impersonation-enable` | Whether to enable impersonation for
the Hadoop catalog fileset. | The
parent(schema) value | No | 0.6.0-incubating |
-| `authentication.type` | The type of authentication for
Hadoop catalog fileset, currently we only support `kerberos`, `simple`. | The
parent(schema) value | No | 0.6.0-incubating |
-| `authentication.kerberos.principal` | The principal of the Kerberos
authentication for the fileset. | The
parent(schema) value | No | 0.6.0-incubating |
-| `authentication.kerberos.keytab-uri` | The URI of The keytab for the
Kerberos authentication for the fileset. | The
parent(schema) value | No | 0.6.0-incubating |
-| `credential-providers` | The credential provider types,
separated by comma. |
(none) | No | 0.8.0-incubating |
-| `placeholder-` | Properties that start with
`placeholder-` are used to replace placeholders in the location. |
(none) | No | 0.9.0-incubating |
+| Property name | Description
| Default
value | Required | Immutable | Since Version |
+|---------------------------------------|--------------------------------------------------------------------------------------------------------|--------------------------|----------|-----------|------------------|
+| `authentication.impersonation-enable` | Whether to enable impersonation for
the Hadoop catalog fileset. | The
parent(schema) value | No | Yes | 0.6.0-incubating |
+| `authentication.type` | The type of authentication for
Hadoop catalog fileset, currently we only support `kerberos`, `simple`. | The
parent(schema) value | No | No | 0.6.0-incubating |
+| `authentication.kerberos.principal` | The principal of the Kerberos
authentication for the fileset. | The
parent(schema) value | No | No | 0.6.0-incubating |
+| `authentication.kerberos.keytab-uri` | The URI of The keytab for the
Kerberos authentication for the fileset. | The
parent(schema) value | No | No | 0.6.0-incubating |
+| `credential-providers` | The credential provider types,
separated by comma. |
(none) | No | No | 0.8.0-incubating |
+| `placeholder-` | Properties that start with
`placeholder-` are used to replace placeholders in the location. |
(none) | No | Yes | 0.9.0-incubating |
Some properties are reserved and cannot be set by users:
-| Property name | Description | Default
value | Since Version |
-|-----------------------|---------------------------------------|-----------------------------|------------------|
-| `placeholder-catalog` | The placeholder for the catalog name. | catalog name
of the fileset | 0.9.0-incubating |
-| `placeholder-schema` | The placeholder for the schema name. | schema name
of the fileset | 0.9.0-incubating |
-| `placeholder-fileset` | The placeholder for the fileset name. | fileset name
| 0.9.0-incubating |
+| Property name | Description
| Default value
| Since Version |
+|-----------------------|----------------------------------------------------------------------------------------------------------|-----------------------------|------------------|
+| `placeholder-catalog` | The placeholder for the catalog name.
| catalog name of the
fileset | 0.9.0-incubating |
+| `placeholder-schema` | The placeholder for the schema name.
| schema name of the
fileset | 0.9.0-incubating |
+| `placeholder-fileset` | The placeholder for the fileset name.
| fileset name
| 0.9.0-incubating |
Credential providers can be specified in several places, as listed below.
Gravitino checks the `credential-providers` setting in the following order of
precedence:
