This is an automated email from the ASF dual-hosted git repository.
rpuch pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new d4dc59000b6 [IGNITE-23326] Validate configuration for duplicated keys
(#5166)
d4dc59000b6 is described below
commit d4dc59000b6a6a4962d501480f656b76879d0fd5
Author: Phillippko <[email protected]>
AuthorDate: Thu Feb 6 18:53:59 2025 +0700
[IGNITE-23326] Validate configuration for duplicated keys (#5166)
---
.../cluster/management/ClusterInitializer.java | 23 +-
.../ConfigurationValidationException.java | 13 +-
.../configuration/validation/ValidationIssue.java | 21 ++
.../ConfigurationDuplicatesValidator.java | 329 +++++++++++++++++++++
.../ConfigurationDuplicatesValidatorTest.java | 109 +++++++
.../configuration/ItClusterConfigurationTest.java | 65 ++++
.../ItNodeBootstrapConfigurationTest.java | 17 ++
.../storage/LocalFileConfigurationStorage.java | 11 +-
.../storage/LocalFileConfigurationStorageTest.java | 31 ++
...ilablePartitionsRecoveryByFilterUpdateTest.java | 2 +-
10 files changed, 604 insertions(+), 17 deletions(-)
diff --git
a/modules/cluster-management/src/main/java/org/apache/ignite/internal/cluster/management/ClusterInitializer.java
b/modules/cluster-management/src/main/java/org/apache/ignite/internal/cluster/management/ClusterInitializer.java
index 96662b60904..dbdb3c6bfbe 100644
---
a/modules/cluster-management/src/main/java/org/apache/ignite/internal/cluster/management/ClusterInitializer.java
+++
b/modules/cluster-management/src/main/java/org/apache/ignite/internal/cluster/management/ClusterInitializer.java
@@ -36,6 +36,7 @@ import
org.apache.ignite.internal.cluster.management.network.messages.CmgInitMes
import
org.apache.ignite.internal.cluster.management.network.messages.CmgMessagesFactory;
import
org.apache.ignite.internal.cluster.management.network.messages.InitCompleteMessage;
import
org.apache.ignite.internal.cluster.management.network.messages.InitErrorMessage;
+import
org.apache.ignite.internal.configuration.validation.ConfigurationDuplicatesValidator;
import
org.apache.ignite.internal.configuration.validation.ConfigurationValidator;
import org.apache.ignite.internal.logger.IgniteLogger;
import org.apache.ignite.internal.logger.Loggers;
@@ -138,15 +139,16 @@ public class ClusterInitializer {
LOG.info("Resolved CMG nodes[nodes={}]", cmgNodes);
- String initialClusterConfiguration =
patchClusterConfigurationWithDynamicDefaults(clusterConfiguration);
- validateConfiguration(initialClusterConfiguration);
+ String patchedClusterConfiguration =
patchClusterConfigurationWithDynamicDefaults(clusterConfiguration);
+
+ validateConfiguration(patchedClusterConfiguration,
clusterConfiguration);
CmgInitMessage initMessage = msgFactory.cmgInitMessage()
.metaStorageNodes(msNodeNameSet)
.cmgNodes(cmgNodeNameSet)
.clusterName(clusterName)
.clusterId(UUID.randomUUID())
- .initialClusterConfiguration(initialClusterConfiguration)
+ .initialClusterConfiguration(patchedClusterConfiguration)
.build();
return invokeMessage(cmgNodes, initMessage)
@@ -261,12 +263,15 @@ public class ClusterInitializer {
return
configurationDynamicDefaultsPatcher.patchWithDynamicDefaults(hocon == null ? ""
: hocon);
}
- private void validateConfiguration(String hocon) {
- if (hocon != null) {
- List<ValidationIssue> issues =
clusterConfigurationValidator.validateHocon(hocon);
- if (!issues.isEmpty()) {
- throw new ConfigurationValidationException(issues);
- }
+ private void validateConfiguration(String patchedHocon, @Nullable String
initialHocon) {
+ List<ValidationIssue> issues =
clusterConfigurationValidator.validateHocon(patchedHocon);
+
+ if (initialHocon != null) {
+
issues.addAll(ConfigurationDuplicatesValidator.validate(initialHocon));
+ }
+
+ if (!issues.isEmpty()) {
+ throw new ConfigurationValidationException(issues);
}
}
}
diff --git
a/modules/configuration-api/src/main/java/org/apache/ignite/configuration/validation/ConfigurationValidationException.java
b/modules/configuration-api/src/main/java/org/apache/ignite/configuration/validation/ConfigurationValidationException.java
index 8edf8959809..5f3aba72972 100644
---
a/modules/configuration-api/src/main/java/org/apache/ignite/configuration/validation/ConfigurationValidationException.java
+++
b/modules/configuration-api/src/main/java/org/apache/ignite/configuration/validation/ConfigurationValidationException.java
@@ -17,6 +17,7 @@
package org.apache.ignite.configuration.validation;
+import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
@@ -24,8 +25,8 @@ import java.util.stream.Collectors;
* Configuration validation exception.
*/
public class ConfigurationValidationException extends RuntimeException {
- /** List of configuration validation issues. */
- private List<ValidationIssue> issues;
+ /** Collection of configuration validation issues. */
+ private Collection<ValidationIssue> issues;
/**
* Constructor.
@@ -41,10 +42,10 @@ public class ConfigurationValidationException extends
RuntimeException {
*
* @param issues List of issues occurred during validation.
*/
- public ConfigurationValidationException(List<ValidationIssue> issues) {
+ public ConfigurationValidationException(Collection<ValidationIssue>
issues) {
super(createMessageFromIssues(issues));
- this.issues = issues;
+ this.issues = List.copyOf(issues);
}
/**
@@ -52,11 +53,11 @@ public class ConfigurationValidationException extends
RuntimeException {
*
* @return List of issues occurred during validation.
*/
- public List<ValidationIssue> getIssues() {
+ public Collection<ValidationIssue> getIssues() {
return issues;
}
- private static String createMessageFromIssues(List<ValidationIssue>
issues) {
+ private static String createMessageFromIssues(Collection<ValidationIssue>
issues) {
return "Validation did not pass for keys: "
+ issues.stream().map(issue -> "[" + issue.key() + ", " +
issue.message() + "]").collect(Collectors.joining(", "));
}
diff --git
a/modules/configuration-api/src/main/java/org/apache/ignite/configuration/validation/ValidationIssue.java
b/modules/configuration-api/src/main/java/org/apache/ignite/configuration/validation/ValidationIssue.java
index d78bb28a1e2..0fd598c1ddf 100644
---
a/modules/configuration-api/src/main/java/org/apache/ignite/configuration/validation/ValidationIssue.java
+++
b/modules/configuration-api/src/main/java/org/apache/ignite/configuration/validation/ValidationIssue.java
@@ -17,6 +17,7 @@
package org.apache.ignite.configuration.validation;
+import java.util.Objects;
import org.apache.ignite.internal.tostring.IgniteToStringInclude;
import org.apache.ignite.internal.tostring.S;
@@ -65,4 +66,24 @@ public class ValidationIssue {
public String toString() {
return S.toString(ValidationIssue.class, this);
}
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+
+ ValidationIssue that = (ValidationIssue) obj;
+
+ return key.equals(that.key) && message.equals(that.message);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(key, message);
+ }
}
diff --git
a/modules/configuration/src/main/java/org/apache/ignite/internal/configuration/validation/ConfigurationDuplicatesValidator.java
b/modules/configuration/src/main/java/org/apache/ignite/internal/configuration/validation/ConfigurationDuplicatesValidator.java
new file mode 100644
index 00000000000..1c252757f6e
--- /dev/null
+++
b/modules/configuration/src/main/java/org/apache/ignite/internal/configuration/validation/ConfigurationDuplicatesValidator.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.configuration.validation;
+
+import com.typesafe.config.ConfigParseOptions;
+import com.typesafe.config.impl.Parseable;
+import com.typesafe.config.parser.ConfigDocument;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.ArrayDeque;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Queue;
+import java.util.Set;
+import org.apache.ignite.configuration.validation.ValidationIssue;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Validates that there are no duplicate keys in the configuration.
+ *
+ * <p>Imagine configuration like this:
+ * <pre>
+ * root {
+ * arrayField: [
+ * arrayMember1: { name = "123" },
+ * arrayMember2: { name = "234" }
+ * ]
+ * }
+ * </pre>
+ *
+ * <p>This configuration can be represented like this:</p>
+ * <ul>
+ * <li>root children: [ ConfigNodePath("root"),
ConfigNodeComplexValue(field) ]</li>
+ * <li>
+ * arrayField children: [ ConfigNodePath("arrayField"),
ConfigNodeArray(arrayMember1), ConfigNodeArray(arrayMember2) ]
+ * </li>
+ * <li>arrayMember1 children: [ ConfigNodeComplexValue(name) ]</li>
+ * <li>arrayMember2 children: [ ConfigNodeComplexValue(name) ]</li>
+ * <li>name children: [ ConfigNodePath("name"), Value("123") ]</li>
+ * <li>name children: [ ConfigNodePath("name"), Value("234") ]</li>
+ * </ul>
+ */
+public class ConfigurationDuplicatesValidator {
+ /**
+ * Validates that there are no duplicate keys in the passed configuration.
+ *
+ * @param cfg configuration in HOCON or JSON format.
+ */
+ public static Collection<ValidationIssue> validate(String cfg) {
+ Object root = getConfigRoot(cfg);
+
+ Queue<Node> queue = new ArrayDeque<>();
+ queue.add(new Node(root, null, null));
+
+ Set<String> paths = new HashSet<>();
+ Set<ValidationIssue> issues = new HashSet<>();
+
+ while (!queue.isEmpty()) {
+ Node currentNode = queue.poll();
+ List<Object> nodeChildren =
getChildrenOrEmpty(currentNode.configNode);
+
+ if (nodeChildren.isEmpty()) {
+ continue;
+ }
+
+ Path currentPath = ConfigNodePath.path(currentNode.basePath,
nodeChildren, currentNode.indexInArray);
+
+ if (currentPath != null) {
+ String pathString = currentPath.toString();
+
+ if (!paths.add(pathString)) {
+ issues.add(new ValidationIssue(pathString, "Duplicated
key"));
+ }
+ } else {
+ currentPath = currentNode.basePath;
+ }
+
+ int index = 0;
+ for (Object child : nodeChildren) {
+ if (ConfigNodeComplexValue.isInstance(child) ||
ConfigNodeField.isInstance(child)) {
+ Integer indexInArray =
ConfigNodeArray.isInstance(currentNode.configNode)
+ ? index
+ : null;
+
+ queue.add(new Node(child, currentPath, indexInArray));
+ index++;
+ }
+ }
+ }
+
+ return issues;
+ }
+
+ private static Object getConfigRoot(String cfg) {
+ ConfigDocument configDocument = Parseable.newString(cfg,
ConfigParseOptions.defaults()).parseConfigDocument();
+ assert SimpleConfigDocument.isInstance(configDocument);
+
+ Object root = SimpleConfigDocument.root(configDocument);
+ assert ConfigNodeComplexValue.isInstance(root);
+
+ return root;
+ }
+
+ private static List<Object> getChildrenOrEmpty(Object configNode) {
+ if (ConfigNodeComplexValue.isInstance(configNode)) {
+ return ConfigNodeComplexValue.children(configNode);
+ } else if (ConfigNodeField.isInstance(configNode)) {
+ return ConfigNodeField.children(configNode);
+ } else {
+ return List.of();
+ }
+ }
+
+ private static class ConfigNodeComplexValue {
+ private static final Class<?> CLASS;
+ private static final Field CHILDREN_FIELD;
+
+ static {
+ try {
+ CLASS =
Class.forName("com.typesafe.config.impl.ConfigNodeComplexValue");
+
+ CHILDREN_FIELD = CLASS.getDeclaredField("children");
+ CHILDREN_FIELD.setAccessible(true);
+ } catch (Exception e) {
+ // Shouldn't happen unless library structure is changed
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static boolean isInstance(Object configNode) {
+ return CLASS.isInstance(configNode);
+ }
+
+ private static List<Object> children(Object configNodeComplexValue) {
+ try {
+ return (List<Object>)
CHILDREN_FIELD.get(configNodeComplexValue);
+ } catch (IllegalAccessException e) {
+ // Shouldn't happen
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static class ConfigNodeArray {
+ private static final Class<?> CLASS;
+
+ static {
+ try {
+ CLASS =
Class.forName("com.typesafe.config.impl.ConfigNodeArray");
+ } catch (Exception e) {
+ // Shouldn't happen unless library structure is changed
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static boolean isInstance(Object configNode) {
+ return CLASS.isInstance(configNode);
+ }
+ }
+
+ private static class ConfigNodeField {
+ private static final Class<?> CLASS;
+ private static final Field CHILDREN_FIELD;
+
+ static {
+ try {
+ CLASS =
Class.forName("com.typesafe.config.impl.ConfigNodeField");
+
+ CHILDREN_FIELD = CLASS.getDeclaredField("children");
+ CHILDREN_FIELD.setAccessible(true);
+ } catch (Exception e) {
+ // Shouldn't happen unless library structure is changed
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static boolean isInstance(Object configNode) {
+ return CLASS.isInstance(configNode);
+ }
+
+ private static List<Object> children(Object configNode) {
+ try {
+ return (List<Object>) CHILDREN_FIELD.get(configNode);
+ } catch (IllegalAccessException e) {
+ // Shouldn't happen
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static class SimpleConfigDocument {
+ private static final Class<?> CLASS;
+ private static final Field NODE_TREE_FIELD;
+
+ static {
+ try {
+ CLASS =
Class.forName("com.typesafe.config.impl.SimpleConfigDocument");
+
+ NODE_TREE_FIELD = CLASS.getDeclaredField("configNodeTree");
+ NODE_TREE_FIELD.setAccessible(true);
+ } catch (Exception e) {
+ // Shouldn't happen unless library structure is changed
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static boolean isInstance(Object configDocument) {
+ return CLASS.isInstance(configDocument);
+ }
+
+ private static Object root(Object simpleConfigDocument) {
+ try {
+ return NODE_TREE_FIELD.get(simpleConfigDocument);
+ } catch (IllegalAccessException e) {
+ // Shouldn't happen
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static class ConfigNodePath {
+ private static final Class<?> CLASS;
+ private static final Class<?> PATH_CLASS;
+ private static final Field PATH_FIELD;
+ private static final Method PATH_RENDER_METHOD;
+
+ static {
+ try {
+ CLASS =
Class.forName("com.typesafe.config.impl.ConfigNodePath");
+ PATH_CLASS = Class.forName("com.typesafe.config.impl.Path");
+
+ PATH_FIELD = CLASS.getDeclaredField("path");
+ PATH_FIELD.setAccessible(true);
+
+ PATH_RENDER_METHOD = PATH_CLASS.getDeclaredMethod("render");
+ PATH_RENDER_METHOD.setAccessible(true);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static boolean isInstance(Object configNode) {
+ return CLASS.isInstance(configNode);
+ }
+
+ /** Returns full path to the given node. Elements of array don't have
a NodePath child, so index is used instead. */
+ private static @Nullable Path path(@Nullable Path basePath,
List<Object> children, @Nullable Integer index) {
+ if (index != null) {
+ return new Path(basePath, "[" + index + "]");
+ }
+
+ Object pathNode = null;
+
+ for (Object child : children) {
+ if (isInstance(child)) {
+ pathNode = child;
+ }
+ }
+
+ if (pathNode == null) {
+ return null;
+ }
+
+ try {
+ Object path = PATH_FIELD.get(pathNode);
+
+ return new Path(basePath,
PATH_RENDER_METHOD.invoke(path).toString());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static class Node {
+ private final Object configNode;
+ private final @Nullable Path basePath;
+ private final @Nullable Integer indexInArray;
+
+ private Node(Object configNode, @Nullable Path basePath, @Nullable
Integer indexInArray) {
+ this.configNode = configNode;
+ this.basePath = basePath;
+ this.indexInArray = indexInArray;
+ }
+ }
+
+ private static class Path {
+ private final @Nullable Path basePath;
+ private final String path;
+
+ private Path(@Nullable Path basePath, String path) {
+ this.basePath = basePath;
+ this.path = path;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+
+ appendTo(builder);
+
+ return builder.toString();
+ }
+
+ private void appendTo(StringBuilder builder) {
+ if (basePath != null) {
+ basePath.appendTo(builder);
+ builder.append('.');
+ }
+
+ builder.append(path);
+ }
+ }
+}
diff --git
a/modules/configuration/src/test/java/org/apache/ignite/internal/configuration/validation/ConfigurationDuplicatesValidatorTest.java
b/modules/configuration/src/test/java/org/apache/ignite/internal/configuration/validation/ConfigurationDuplicatesValidatorTest.java
new file mode 100644
index 00000000000..9d441a773a8
--- /dev/null
+++
b/modules/configuration/src/test/java/org/apache/ignite/internal/configuration/validation/ConfigurationDuplicatesValidatorTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.configuration.validation;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.empty;
+
+import org.apache.ignite.configuration.validation.ValidationIssue;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.junit.jupiter.api.Test;
+
+class ConfigurationDuplicatesValidatorTest extends BaseIgniteAbstractTest {
+ private static final String DUPLICATE_CONFIG = "ignite {\n"
+ + " network: {\n"
+ + " port: 456,\n"
+ + " nodeFinder.netClusterNodes: [ \"localhost\" ]\n"
+ + " },\n"
+ + " network: {}\n"
+ + " network.nodeFinder.netClusterNodes: [ \"localhost\" ],\n"
+ + " storage.profiles: {\n"
+ + " test.engine: test, \n"
+ + " default_aipersist.engine: aipersist, \n"
+ + " default_aipersist.size: 123, \n"
+ + " test.engine: aimem \n"
+ + " },\n"
+ + " storage {\n"
+ + " profiles = [\n"
+ + " {\n"
+ + " name = persistent,\n"
+ + " name = persistent\n"
+ + " }\n"
+ + " ]\n"
+ + " },\n"
+ + " clientConnector.port: 123,\n"
+ + " rest {\n"
+ + " port: 123,\n"
+ + " port: 456\n"
+ + " },\n"
+ + " compute { threadPoolSize: 1 },\n"
+ + " compute { threadPoolSize: 2 }\n"
+ + " failureHandler.dumpThreadsOnFailure: false\n"
+ + "}";
+
+ private static final String OK_CONFIG = "ignite {\n"
+ + " network: {\n"
+ + " port: 456,\n"
+ + " nodeFinder.netClusterNodes: [ {} ]\n"
+ + " },\n"
+ + " storage {\n"
+ + " profiles = [\n"
+ + " {\n"
+ + " name = persistent,\n"
+ + " engine = aipersist\n"
+ + " }\n"
+ + " {\n"
+ + " name = rocksdb-example,\n"
+ + " engine = rocksdb\n"
+ + " }\n"
+ + " {\n"
+ + " name = in-memory,\n"
+ + " engine = aimem\n"
+ + " }\n"
+ + " ]\n"
+ + " },\n"
+ + " clientConnector.port: 123,\n"
+ + " rest {\n"
+ + " port: 123\n"
+ + " },\n"
+ + " compute.threadPoolSize: 1,\n"
+ + " failureHandler.dumpThreadsOnFailure: false\n"
+ + "}";
+
+ @Test
+ void testValidationSucceededWithNoDuplicates() {
+ assertThat(ConfigurationDuplicatesValidator.validate(OK_CONFIG),
empty());
+ }
+
+ @Test
+ void testValidationFailedWithDuplicates() {
+ assertThat(ConfigurationDuplicatesValidator.validate(DUPLICATE_CONFIG),
+ containsInAnyOrder(
+ new ValidationIssue("ignite.network", "Duplicated
key"),
+ new
ValidationIssue("ignite.network.nodeFinder.netClusterNodes", "Duplicated key"),
+ new
ValidationIssue("ignite.storage.profiles.[0].name", "Duplicated key"),
+ new ValidationIssue("ignite.storage.profiles",
"Duplicated key"),
+ new
ValidationIssue("ignite.storage.profiles.test.engine", "Duplicated key"),
+ new ValidationIssue("ignite.rest.port", "Duplicated
key"),
+ new ValidationIssue("ignite.compute", "Duplicated
key"),
+ new ValidationIssue("ignite.compute.threadPoolSize",
"Duplicated key")
+ )
+ );
+ }
+}
diff --git
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/configuration/ItClusterConfigurationTest.java
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/configuration/ItClusterConfigurationTest.java
new file mode 100644
index 00000000000..d58fcfa500a
--- /dev/null
+++
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/configuration/ItClusterConfigurationTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.configuration;
+
+import static
org.apache.ignite.internal.testframework.IgniteTestUtils.assertThrowsWithCause;
+import static
org.apache.ignite.internal.testframework.IgniteTestUtils.testNodeName;
+
+import java.nio.file.Path;
+import org.apache.ignite.IgniteServer;
+import org.apache.ignite.InitParameters;
+import
org.apache.ignite.configuration.validation.ConfigurationValidationException;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.internal.testframework.TestIgnitionManager;
+import org.apache.ignite.internal.testframework.WorkDirectory;
+import org.apache.ignite.internal.testframework.WorkDirectoryExtension;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/**
+ * Integration test for checking Cluster configuration.
+ */
+@ExtendWith(WorkDirectoryExtension.class)
+public class ItClusterConfigurationTest extends BaseIgniteAbstractTest {
+ private IgniteServer node;
+
+ @AfterEach
+ void tearDown() {
+ if (node != null) {
+ node.shutdown();
+ }
+ }
+
+ @Test
+ void testValidationFailsWithDuplicates(TestInfo testInfo, @WorkDirectory
Path workDir) {
+ node = TestIgnitionManager.start(testNodeName(testInfo, 0), null,
workDir);
+
+ var parameters = InitParameters.builder()
+ .metaStorageNodes(node)
+ .clusterName("cluster")
+ .clusterConfiguration("ignite { metaStorage {
idleSyncTimeInterval: 50, idleSyncTimeInterval: 100 }}")
+ .build();
+
+ assertThrowsWithCause(
+ () -> TestIgnitionManager.init(node, parameters),
+ ConfigurationValidationException.class,
+ "Validation did not pass for keys:
[ignite.metaStorage.idleSyncTimeInterval, Duplicated key]");
+ }
+}
diff --git
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/configuration/ItNodeBootstrapConfigurationTest.java
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/configuration/ItNodeBootstrapConfigurationTest.java
index bb826188e94..0c21dae9db6 100644
---
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/configuration/ItNodeBootstrapConfigurationTest.java
+++
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/configuration/ItNodeBootstrapConfigurationTest.java
@@ -100,4 +100,21 @@ public class ItNodeBootstrapConfigurationTest {
ConfigurationValidationException.class,
"Validation did not pass for keys: [ignite.rest.ssl.keyStore,
Key store file doesn't exist at bad_path]");
}
+
+ @Test
+ public void testConfigurationValidationFailsWithDuplicates(TestInfo
testInfo) {
+ String config = "ignite {\n"
+ + " rest: {\n"
+ + " ssl: {\n"
+ + " enabled: false,\n"
+ + " enabled: true\n"
+ + " }\n"
+ + " }\n"
+ + "}";
+
+ assertThrowsWithCause(
+ () -> TestIgnitionManager.start(testNodeName(testInfo, 0),
config, workDir),
+ ConfigurationValidationException.class,
+ "Validation did not pass for keys: [ignite.rest.ssl.enabled,
Duplicated key]");
+ }
}
diff --git
a/modules/runner/src/main/java/org/apache/ignite/internal/configuration/storage/LocalFileConfigurationStorage.java
b/modules/runner/src/main/java/org/apache/ignite/internal/configuration/storage/LocalFileConfigurationStorage.java
index 96e431c447b..23e532f79f4 100644
---
a/modules/runner/src/main/java/org/apache/ignite/internal/configuration/storage/LocalFileConfigurationStorage.java
+++
b/modules/runner/src/main/java/org/apache/ignite/internal/configuration/storage/LocalFileConfigurationStorage.java
@@ -40,6 +40,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
+import java.util.Collection;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.CompletableFuture;
@@ -54,6 +55,7 @@ import
org.apache.ignite.configuration.ConfigurationDynamicDefaultsPatcher;
import org.apache.ignite.configuration.ConfigurationModule;
import org.apache.ignite.configuration.annotation.ConfigurationType;
import
org.apache.ignite.configuration.validation.ConfigurationValidationException;
+import org.apache.ignite.configuration.validation.ValidationIssue;
import
org.apache.ignite.internal.configuration.ConfigurationDynamicDefaultsPatcherImpl;
import org.apache.ignite.internal.configuration.ConfigurationTreeGenerator;
import org.apache.ignite.internal.configuration.NodeConfigCreateException;
@@ -62,6 +64,7 @@ import
org.apache.ignite.internal.configuration.NodeConfigWriteException;
import org.apache.ignite.internal.configuration.SuperRoot;
import org.apache.ignite.internal.configuration.hocon.HoconConverter;
import org.apache.ignite.internal.configuration.tree.ConverterToMapVisitor;
+import
org.apache.ignite.internal.configuration.validation.ConfigurationDuplicatesValidator;
import org.apache.ignite.internal.future.InFlightFutures;
import org.apache.ignite.internal.logger.IgniteLogger;
import org.apache.ignite.internal.logger.Loggers;
@@ -188,10 +191,16 @@ public class LocalFileConfigurationStorage implements
ConfigurationStorage {
try {
String confString = Files.readString(configPath.toAbsolutePath());
+ Collection<ValidationIssue> duplicates =
ConfigurationDuplicatesValidator.validate(confString);
+
+ if (!duplicates.isEmpty()) {
+ throw new ConfigurationValidationException(duplicates);
+ }
+
ConfigParseOptions parseOptions =
ConfigParseOptions.defaults().setSyntax(ConfigSyntax.CONF).setAllowMissing(false);
return ConfigFactory.parseString(patch(confString, module),
parseOptions);
- } catch (Parse | ConfigurationValidationException | IOException e) {
+ } catch (Parse | IOException e) {
throw new NodeConfigParseException("Failed to parse config content
from file " + configPath, e);
}
}
diff --git
a/modules/runner/src/test/java/org/apache/ignite/internal/configuration/storage/LocalFileConfigurationStorageTest.java
b/modules/runner/src/test/java/org/apache/ignite/internal/configuration/storage/LocalFileConfigurationStorageTest.java
index bae8dc18c6b..ca9fe676b36 100644
---
a/modules/runner/src/test/java/org/apache/ignite/internal/configuration/storage/LocalFileConfigurationStorageTest.java
+++
b/modules/runner/src/test/java/org/apache/ignite/internal/configuration/storage/LocalFileConfigurationStorageTest.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.configuration.storage;
+import static
org.apache.ignite.internal.testframework.IgniteTestUtils.assertThrows;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.aMapWithSize;
import static org.hamcrest.Matchers.allOf;
@@ -44,6 +45,7 @@ import org.apache.ignite.configuration.annotation.ConfigValue;
import org.apache.ignite.configuration.annotation.ConfigurationRoot;
import org.apache.ignite.configuration.annotation.NamedConfigValue;
import org.apache.ignite.configuration.annotation.Value;
+import
org.apache.ignite.configuration.validation.ConfigurationValidationException;
import org.apache.ignite.internal.configuration.ConfigurationTreeGenerator;
import org.apache.ignite.internal.configuration.TestConfigurationChanger;
import
org.apache.ignite.internal.configuration.validation.ConfigurationValidatorImpl;
@@ -536,6 +538,35 @@ public class LocalFileConfigurationStorageTest {
assertDoesNotThrow(storage::readDataOnRecovery);
}
+ @Test
+ void testValidateDuplicates() throws IOException {
+ // Given config in JSON format
+ String fileContent
+ = "top {\n"
+ + " inner {\n"
+ + " boolVal=false,\n"
+ + " boolVal=true\n"
+ + " someConfigurationValue {\n"
+ + " intVal=1\n"
+ + " strVal=foo\n"
+ + " }\n"
+ + " strVal=foo\n"
+ + " }\n"
+ + " shortVal=3\n"
+ + "}\n";
+
+ Path configFile = getConfigFile();
+
+ Files.write(configFile, fileContent.getBytes(StandardCharsets.UTF_8));
+
+ // And storage detects duplicates
+ assertThrows(
+ ConfigurationValidationException.class,
+ changer::start,
+ "Validation did not pass for keys: [top.inner.boolVal,
Duplicated key]"
+ );
+ }
+
private String configFileContent() throws IOException {
return Files.readString(getConfigFile());
}
diff --git
a/modules/table/src/integrationTest/java/org/apache/ignite/internal/table/distributed/disaster/ItHighAvailablePartitionsRecoveryByFilterUpdateTest.java
b/modules/table/src/integrationTest/java/org/apache/ignite/internal/table/distributed/disaster/ItHighAvailablePartitionsRecoveryByFilterUpdateTest.java
index f64ea96a128..18d1a24ed7c 100644
---
a/modules/table/src/integrationTest/java/org/apache/ignite/internal/table/distributed/disaster/ItHighAvailablePartitionsRecoveryByFilterUpdateTest.java
+++
b/modules/table/src/integrationTest/java/org/apache/ignite/internal/table/distributed/disaster/ItHighAvailablePartitionsRecoveryByFilterUpdateTest.java
@@ -132,7 +132,7 @@ public class
ItHighAvailablePartitionsRecoveryByFilterUpdateTest extends Abstrac
) {
return "ignite {\n"
+ " nodeAttributes.nodeAttributes: " + nodeAtrributes + ",\n"
- + " storage.profiles: " + storageProfiles + ",\n"
+ + (storageProfiles == null ? "" : " storage.profiles: " +
storageProfiles + ",\n")
+ " network: {\n"
+ " port: {},\n"
+ " nodeFinder: {\n"
