This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7d231485588 [fix](fe) Validate Hive HDFS HA properties during catalog
DDL (#68585)
7d231485588 is described below
commit 7d2314855883bab61255d77c6c18d8d535357a68
Author: Oliveira <[email protected]>
AuthorDate: Wed Sep 30 16:25:05 2026 +0800
[fix](fe) Validate Hive HDFS HA properties during catalog DDL (#68585)
### What problem does this PR solve?
Issue Number: DORIS-29461
Related PR: None
Problem Summary: Plugin-driven Hive catalogs accepted incomplete HDFS HA
properties during CREATE and ALTER. For example, a catalog with
`dfs.nameservices=ns1` but no `dfs.ha.namenodes.ns1`, or with an empty
nameservice list, could be persisted successfully and only fail when the
first external table accessed storage. The DDL validation path now binds
Hive storage properties before persisting the catalog, and HDFS HA
validation rejects empty nameservice entries and validates the required
NameNode, RPC address, and failover provider properties. The behavior is
covered by FE unit tests and an external regression suite.
### Release note
Hive Catalog CREATE and ALTER now reject incomplete HDFS HA
configurations during DDL validation.
### Check List (For Author)
- Test: Unit Test / Regression test
- FE unit tests: `PluginDrivenExternalCatalogConcurrencyTest`,
`HdfsPropertiesTest`
- External regression:
`external_table_p0/hive/test_hive_ha_catalog_validation`
- FE build and Checkstyle passed
- Behavior changed: Yes (invalid Hive HDFS HA configurations fail during
CREATE/ALTER instead of being deferred until storage access)
- Does this need documentation: No
---
.../org/apache/doris/datasource/CatalogMgr.java | 92 ++++++++++++++++------
.../plugin/PluginDrivenExternalCatalog.java | 25 ++++++
.../doris/datasource/storage/StorageAdapter.java | 2 +-
.../org/apache/doris/fs/FileSystemFactory.java | 14 ++--
...PluginDrivenExternalCatalogConcurrencyTest.java | 30 +++++++
.../hdfs/properties/HdfsConfigFileLoader.java | 3 +-
.../hdfs/properties/HdfsPropertiesUtils.java | 12 ++-
.../hdfs/properties/HdfsPropertiesTest.java | 25 +++++-
.../hive/test_hive_ha_catalog_validation.groovy | 91 +++++++++++++++++++++
9 files changed, 254 insertions(+), 40 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index d3fa79b847c..fb2ce39374d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -288,6 +288,14 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
}
long id = Env.getCurrentEnv().getNextId();
CatalogIf catalog = CatalogFactory.createFromCommand(id, cmd);
+ if (catalog instanceof ExternalCatalog) {
+ try {
+ ((ExternalCatalog) catalog).checkProperties();
+ } catch (DdlException | RuntimeException e) {
+ catalog.onCreateFailure();
+ throw e;
+ }
+ }
createCatalogImpl(catalog, cmd.getCatalogName(),
cmd.isSetIfNotExists());
}
@@ -410,26 +418,60 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
* Modify the catalog property and write the meta log.
*/
public void alterCatalogProps(String catalogName, Map<String, String>
newProperties) throws UserException {
- Runnable accessControllerCleanup = () -> { };
- writeLock();
- try {
- CatalogIf catalog = nameToCatalog.get(catalogName);
- if (catalog == null) {
- throw new DdlException("No catalog found with name: " +
catalogName);
+ Map<String, String> updates = Maps.newHashMap(newProperties);
+ while (true) {
+ CatalogIf catalog;
+ Map<String, String> oldProperties;
+ readLock();
+ try {
+ catalog = nameToCatalog.get(catalogName);
+ if (catalog == null) {
+ throw new DdlException("No catalog found with name: " +
catalogName);
+ }
+ oldProperties = Maps.newHashMap(catalog.getProperties());
+ if (updates.containsKey("type") && !catalog.getType()
+ .equalsIgnoreCase(updates.get("type"))) {
+ throw new DdlException("Can't modify the type of catalog
property with name: " + catalogName);
+ }
+ } finally {
+ readUnlock();
}
- Map<String, String> oldProperties = catalog.getProperties();
- if (newProperties.containsKey("type") && !catalog.getType()
- .equalsIgnoreCase(newProperties.get("type"))) {
- throw new DdlException("Can't modify the type of catalog
property with name: " + catalogName);
+
+ // Filesystem binding may read Hadoop XML files. Validate the
detached snapshot before
+ // taking the global catalog write lock, then retry if another
ALTER changed the snapshot.
+ boolean validatedWithoutMutation = catalog instanceof
ExternalCatalog
+ && validateCatalogPropsBeforeUpdate((ExternalCatalog)
catalog, oldProperties, updates);
+ Runnable accessControllerCleanup = () -> { };
+ writeLock();
+ try {
+ if (nameToCatalog.get(catalogName) != catalog
+ || !oldProperties.equals(catalog.getProperties())) {
+ continue;
+ }
+ CatalogLog log = new CatalogLog();
+ log.setCatalogId(catalog.getId());
+ log.setNewProps(updates);
+ accessControllerCleanup = applyAlterCatalogProps(
+ log, oldProperties, false, true,
validatedWithoutMutation);
+
Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_ALTER_CATALOG_PROPS,
log);
+ return;
+ } finally {
+ writeUnlock();
+ accessControllerCleanup.run();
}
- CatalogLog log = new CatalogLog();
- log.setCatalogId(catalog.getId());
- log.setNewProps(newProperties);
- accessControllerCleanup = applyAlterCatalogProps(log,
oldProperties, false, true);
-
Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_ALTER_CATALOG_PROPS,
log);
- } finally {
- writeUnlock();
- accessControllerCleanup.run();
+ }
+ }
+
+ private boolean validateCatalogPropsBeforeUpdate(ExternalCatalog catalog,
Map<String, String> oldProperties,
+ Map<String, String> newProperties) throws DdlException {
+ try {
+ return catalog.validatePropertiesBeforeUpdate(oldProperties,
newProperties);
+ } catch (Exception validationException) {
+ if (validationException instanceof DdlException) {
+ throw (DdlException) validationException;
+ }
+ throw new DdlException("Invalid catalog properties: "
+ + validationException.getMessage(), validationException);
}
}
@@ -553,9 +595,6 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
private void createCatalogInternal(CatalogIf catalog, boolean isReplay)
throws DdlException {
writeLock();
try {
- if (!isReplay && catalog instanceof ExternalCatalog) {
- ((ExternalCatalog) catalog).checkProperties();
- }
Map<String, String> props = catalog.getProperties();
if (props.containsKey(METADATA_REFRESH_INTERVAL_SEC)) {
// need refresh
@@ -647,7 +686,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
Runnable accessControllerCleanup = () -> { };
writeLock();
try {
- accessControllerCleanup = applyAlterCatalogProps(log,
oldProperties, isReplay, true);
+ accessControllerCleanup = applyAlterCatalogProps(log,
oldProperties, isReplay, true, false);
} finally {
writeUnlock();
accessControllerCleanup.run();
@@ -655,16 +694,17 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
}
private Runnable applyAlterCatalogProps(CatalogLog log, Map<String,
String> oldProperties,
- boolean isReplay, boolean deferAccessControllerCleanup) throws
DdlException {
+ boolean isReplay, boolean deferAccessControllerCleanup, boolean
validatedWithoutMutation)
+ throws DdlException {
CatalogIf catalog = idToCatalog.get(log.getCatalogId());
if (catalog instanceof ExternalCatalog) {
Map<String, String> newProps = log.getNewProps();
- if (!isReplay) {
+ if (!isReplay && !validatedWithoutMutation) {
ExternalCatalog externalCatalog = (ExternalCatalog) catalog;
try {
- boolean validatedWithoutMutation =
externalCatalog.validatePropertiesBeforeUpdate(
+ boolean validatedWithoutMutationInLock =
externalCatalog.validatePropertiesBeforeUpdate(
oldProperties, newProps);
- if (!validatedWithoutMutation) {
+ if (!validatedWithoutMutationInLock) {
synchronized (externalCatalog) {
Map<String, String> currentProperties =
Maps.newHashMap(externalCatalog.getProperties());
try {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java
index 85959852cf5..2a877d26b20 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java
@@ -69,7 +69,9 @@ import
org.apache.doris.datasource.connector.converter.ConnectorColumnConverter;
import
org.apache.doris.datasource.connector.converter.ConnectorPartitionFieldConverter;
import org.apache.doris.datasource.log.ExternalObjectLog;
import org.apache.doris.datasource.log.InitCatalogLog;
+import org.apache.doris.foundation.property.StoragePropertiesException;
import org.apache.doris.foundation.security.JdbcDriverUrlSecurity;
+import org.apache.doris.fs.FileSystemFactory;
import org.apache.doris.nereids.trees.plans.commands.info.AddPartitionFieldOp;
import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo;
import org.apache.doris.nereids.trees.plans.commands.info.DropPartitionFieldOp;
@@ -230,6 +232,7 @@ public class PluginDrivenExternalCatalog extends
ExternalCatalog {
} catch (IllegalArgumentException e) {
throw new DdlException(e.getMessage());
}
+ validateStorageProperties(catalogProperty);
// Validate function_rules JSON if present (shared across all
connector types).
String functionRules = catalogProperty.getOrDefault("function_rules",
null);
ExternalFunctionRules.check(functionRules);
@@ -253,11 +256,33 @@ public class PluginDrivenExternalCatalog extends
ExternalCatalog {
} catch (IllegalArgumentException e) {
throw new DdlException(e.getMessage(), e);
}
+ validateStorageProperties(candidateProperty);
checkDriverUrlsAgainstOperatorGate(candidate, updatedProperties);
ExternalFunctionRules.check(candidateProperty.getOrDefault("function_rules",
null));
return true;
}
+ /**
+ * Bind Hive catalog storage properties during DDL validation so HDFS
configuration errors are
+ * reported by CREATE/ALTER instead of being deferred until the first
table access.
+ */
+ private void validateStorageProperties(CatalogProperty property) throws
DdlException {
+ if (!"hms".equalsIgnoreCase(getType())) {
+ return;
+ }
+ String nameservices = property.getProperties().get("dfs.nameservices");
+ if (nameservices != null
+ && java.util.Arrays.stream(nameservices.split(","))
+ .map(String::trim).noneMatch(value -> !value.isEmpty())) {
+ throw new DdlException("Property dfs.nameservices must contain a
nameservice");
+ }
+ try {
+
FileSystemFactory.bindAllStorageProperties(property.getProperties());
+ } catch (IllegalArgumentException | StoragePropertiesException e) {
+ throw new DdlException(e.getMessage(), e);
+ }
+ }
+
/**
* Applies the operator's driver-jar gate ({@code jdbc_driver_secure_path}
/
* {@code jdbc_driver_url_white_list}) to every driver_url these
properties would make the connector
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageAdapter.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageAdapter.java
index e5df6e21196..0c42c095d33 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageAdapter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageAdapter.java
@@ -206,7 +206,7 @@ public final class StorageAdapter {
* Broker/Local/Http verbatim backend maps stay clean — and only when a
* {@code *config.resources} key is present (the only consumer of the
marker).
*/
- private static Map<String, String> withHadoopConfigDir(Map<String, String>
props) {
+ public static Map<String, String> withHadoopConfigDir(Map<String, String>
props) {
if (props == null || props.containsKey("_HADOOP_CONFIG_DIR_")
|| StringUtils.isBlank(Config.hadoop_config_dir)) {
return props;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
index bbf1ee84e21..a71d6c0a2e4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
@@ -97,18 +97,19 @@ public final class FileSystemFactory {
*/
public static org.apache.doris.filesystem.FileSystem
getFileSystem(Map<String, String> properties)
throws IOException {
+ Map<String, String> normalizedProperties =
StorageAdapter.withHadoopConfigDir(properties);
FileSystemPluginManager mgr = pluginManager;
if (mgr != null) {
- return mgr.createFileSystem(properties);
+ return mgr.createFileSystem(normalizedProperties);
}
// Fallback: ServiceLoader discovery (unit-test / migration path)
List<FileSystemProvider> providers = getProviders();
List<String> tried = new ArrayList<>();
for (FileSystemProvider provider : providers) {
- if (provider.supports(properties)) {
+ if (provider.supports(normalizedProperties)) {
LOG.debug("FileSystemFactory: selected SPI provider '{}' for
keys={}",
provider.name(), properties.keySet());
- return provider.create(properties);
+ return provider.create(normalizedProperties);
}
tried.add(provider.name());
}
@@ -135,16 +136,17 @@ public final class FileSystemFactory {
// fe-core Config, so the HDFS plugin's config-resource loader reads
this system property instead. Keep
// the key in sync with HdfsConfigFileLoader.CONFIG_DIR_PROPERTY
("doris.hadoop.config.dir").
System.setProperty("doris.hadoop.config.dir",
Config.hadoop_config_dir);
+ Map<String, String> normalizedProperties =
StorageAdapter.withHadoopConfigDir(properties);
FileSystemPluginManager mgr = pluginManager;
if (mgr != null) {
- return new ArrayList<>(mgr.bindAll(properties));
+ return new ArrayList<>(mgr.bindAll(normalizedProperties));
}
// Fallback: ServiceLoader discovery (unit-test / migration path),
mirroring getFileSystem(Map).
List<org.apache.doris.filesystem.properties.StorageProperties> result
= new ArrayList<>();
for (FileSystemProvider provider : getProviders()) {
- if (provider.supports(properties)) {
+ if (provider.supports(normalizedProperties)) {
try {
- result.add(provider.bind(properties));
+ result.add(provider.bind(normalizedProperties));
} catch (UnsupportedOperationException e) {
LOG.debug("FileSystemProvider {} has no typed binding;
skipping in "
+ "bindAllStorageProperties", provider.name());
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java
index 0b456b2cebb..2896ac58f58 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java
@@ -100,6 +100,36 @@ public class PluginDrivenExternalCatalogConcurrencyTest {
Assertions.assertTrue(exception.getMessage().contains("short timezone
aliases are not supported"));
}
+ @Test
+ public void testCreateRejectsIncompleteHiveHaConfiguration() {
+ Map<String, String> properties = hiveCatalogProperties();
+ properties.put("dfs.nameservices", "ns1");
+ PluginDrivenExternalCatalog catalog = new PluginDrivenExternalCatalog(
+ 1L, "test-catalog", null, properties, "", null);
+
+ DdlException exception = Assertions.assertThrows(DdlException.class,
catalog::checkProperties);
+
Assertions.assertTrue(exception.getMessage().contains("dfs.ha.namenodes.ns1"),
exception.getMessage());
+ }
+
+ @Test
+ public void testAlterRejectsIncompleteHiveHaConfiguration() {
+ Map<String, String> properties = hiveCatalogProperties();
+ PluginDrivenExternalCatalog catalog = new PluginDrivenExternalCatalog(
+ 1L, "test-catalog", null, properties, "", null);
+
+ DdlException exception = Assertions.assertThrows(DdlException.class,
+ () -> catalog.validatePropertiesBeforeUpdate(properties,
+ Collections.singletonMap("dfs.nameservices", "ns1")));
+
Assertions.assertTrue(exception.getMessage().contains("dfs.ha.namenodes.ns1"),
exception.getMessage());
+ }
+
+ private static Map<String, String> hiveCatalogProperties() {
+ Map<String, String> properties = new HashMap<>();
+ properties.put("type", "hms");
+ properties.put("hive.metastore.uris", "thrift://localhost:9083");
+ return properties;
+ }
+
@Test
public void testPropertyUpdateInvalidatesCatalogCaches() throws Exception {
TestablePluginCatalog catalog = new TestablePluginCatalog(
diff --git
a/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsConfigFileLoader.java
b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsConfigFileLoader.java
index 49dedf4173f..d45fd8365d6 100644
---
a/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsConfigFileLoader.java
+++
b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsConfigFileLoader.java
@@ -46,7 +46,8 @@ public final class HdfsConfigFileLoader {
String dir = configDir == null ? "" : configDir;
Configuration conf = new Configuration();
for (String resource : resourcesPath.split(",")) {
- String path = dir + resource.trim();
+ String resourceName = resource.trim();
+ String path = new File(resourceName).isAbsolute() ? resourceName :
dir + resourceName;
File file = new File(path);
if (file.exists() && file.isFile()) {
conf.addResource(new Path(file.toURI()));
diff --git
a/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsPropertiesUtils.java
b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsPropertiesUtils.java
index 18972d59eb2..ffea39df0ab 100644
---
a/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsPropertiesUtils.java
+++
b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsPropertiesUtils.java
@@ -218,10 +218,14 @@ public class HdfsPropertiesUtils {
// No nameservice configured => HA is not enabled, nothing to
validate
return;
}
- for (String dfsservice : splitAndTrim(dfsNameservices)) {
- if (dfsservice.isEmpty()) {
- continue;
- }
+ // Keep runtime binding compatible with catalogs written by older
versions. Hadoop also
+ // ignores empty tokens produced by a trailing comma in
dfs.nameservices.
+ List<String> services = splitAndTrim(dfsNameservices);
+ if (services.isEmpty()) {
+ // Keep runtime binding compatible with catalogs persisted by
older versions.
+ return;
+ }
+ for (String dfsservice : services) {
// 2. Check dfs.ha.namenodes.<nameservice>
String haNnKey = HdfsClientConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX
+ "." + dfsservice;
String namenodes = hdfsProperties.getOrDefault(haNnKey, "");
diff --git
a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/properties/HdfsPropertiesTest.java
b/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/properties/HdfsPropertiesTest.java
index 5785ff84e99..e224054c642 100644
---
a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/properties/HdfsPropertiesTest.java
+++
b/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/properties/HdfsPropertiesTest.java
@@ -63,6 +63,27 @@ class HdfsPropertiesTest {
Assertions.assertFalse(HdfsConfigBuilder.isKerberosEnabled(resolved));
}
+ @Test
+ void emptyNameserviceListRemainsCompatibleAtRuntime() {
+ Map<String, String> raw = new HashMap<>();
+ raw.put("dfs.nameservices", ",");
+
+ Assertions.assertDoesNotThrow(() -> resolve(raw));
+ }
+
+ @Test
+ void nameserviceListWithTrailingEmptyEntryRemainsCompatible() {
+ Map<String, String> raw = new HashMap<>();
+ raw.put("dfs.nameservices", "ns1,");
+ raw.put("dfs.ha.namenodes.ns1", "nn1,nn2");
+ raw.put("dfs.namenode.rpc-address.ns1.nn1", "host1:8020");
+ raw.put("dfs.namenode.rpc-address.ns1.nn2", "host2:8020");
+ raw.put("dfs.client.failover.proxy.provider.ns1",
+
"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
+
+ Assertions.assertDoesNotThrow(() -> resolve(raw));
+ }
+
@Test
void userOverriddenHadoopKeysArePreserved() {
Map<String, String> raw = new HashMap<>();
@@ -102,8 +123,8 @@ class HdfsPropertiesTest {
Map<String, String> raw = new HashMap<>();
raw.put("fs.defaultFS", "hdfs://ns");
- // No _HADOOP_CONFIG_DIR_ injected: the resource is given as a full
absolute path
- // and must load as-is (configDir falls back to "").
+ // An absolute resource must remain absolute even when the configured
Hadoop directory is set.
+ raw.put("_HADOOP_CONFIG_DIR_", tmp.resolve("configured").toString() +
"/");
raw.put("hadoop.config.resources", site.toString());
Map<String, String> resolved = resolve(raw);
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_ha_catalog_validation.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_ha_catalog_validation.groovy
new file mode 100644
index 00000000000..2ee9ada8005
--- /dev/null
+++
b/regression-test/suites/external_table_p0/hive/test_hive_ha_catalog_validation.groovy
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+suite("test_hive_ha_catalog_validation", "p0,external") {
+ sql "drop catalog if exists test_hive_ha_catalog_validation"
+
+ test {
+ sql """create catalog test_hive_ha_catalog_validation properties (
+ 'type' = 'hms',
+ 'hive.metastore.uris' = 'thrift://127.0.0.1:9083',
+ 'test_connection' = 'false',
+ 'dfs.nameservices' = 'ns1'
+ )"""
+ exception "Missing property: dfs.ha.namenodes.ns1"
+ }
+
+ test {
+ sql """create catalog test_hive_ha_catalog_validation properties (
+ 'type' = 'hms',
+ 'hive.metastore.uris' = 'thrift://127.0.0.1:9083',
+ 'test_connection' = 'false',
+ 'dfs.nameservices' = ','
+ )"""
+ exception "dfs.nameservices must contain a nameservice"
+ }
+
+ test {
+ sql """create catalog test_hive_ha_catalog_validation properties (
+ 'type' = 'hms',
+ 'hive.metastore.uris' = 'thrift://127.0.0.1:9083',
+ 'test_connection' = 'false',
+ 'dfs.nameservices' = 'ns1,'
+ )"""
+ exception "Missing property: dfs.ha.namenodes.ns1"
+ }
+
+ test {
+ sql """create catalog test_hive_ha_catalog_validation properties (
+ 'type' = 'hms',
+ 'hive.metastore.uris' = 'thrift://127.0.0.1:9083',
+ 'test_connection' = 'false',
+ 'dfs.nameservices' = 'ns1',
+ 'dfs.ha.namenodes.ns1' = 'nn1,nn2'
+ )"""
+ exception "Missing property: dfs.namenode.rpc-address.ns1.nn1"
+ }
+
+ test {
+ sql """create catalog test_hive_ha_catalog_validation properties (
+ 'type' = 'hms',
+ 'hive.metastore.uris' = 'thrift://127.0.0.1:9083',
+ 'test_connection' = 'false',
+ 'dfs.nameservices' = 'ns1',
+ 'dfs.ha.namenodes.ns1' = 'nn1,nn2',
+ 'dfs.namenode.rpc-address.ns1.nn1' = '127.0.0.1:8020',
+ 'dfs.namenode.rpc-address.ns1.nn2' = '127.0.0.1:8021'
+ )"""
+ exception "Missing property: dfs.client.failover.proxy.provider.ns1"
+ }
+
+ sql """create catalog test_hive_ha_catalog_validation properties (
+ 'type' = 'hms',
+ 'hive.metastore.uris' = 'thrift://127.0.0.1:9083',
+ 'test_connection' = 'false',
+ 'dfs.nameservices' = 'ns1',
+ 'dfs.ha.namenodes.ns1' = 'nn1,nn2',
+ 'dfs.namenode.rpc-address.ns1.nn1' = '127.0.0.1:8020',
+ 'dfs.namenode.rpc-address.ns1.nn2' = '127.0.0.1:8021',
+ 'dfs.client.failover.proxy.provider.ns1' =
'org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider'
+ )"""
+
+ test {
+ sql """alter catalog test_hive_ha_catalog_validation set properties (
+ 'dfs.nameservices' = 'ns2'
+ )"""
+ exception "Missing property: dfs.ha.namenodes.ns2"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
