This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new fdb1370b76 Added servers per host to cluster.yaml (#2903)
fdb1370b76 is described below
commit fdb1370b76ca562babc5c3ff67376f0ca332a8d9
Author: Dave Marion <dlmar...@apache.org>
AuthorDate: Wed Aug 31 14:16:27 2022 -0400
Added servers per host to cluster.yaml (#2903)
Added tservers_per_host and sservers_per_host to cluster.yaml file. This
file is parsed
and transformed into environment variables that are read in by the
accumulo-cluster script.
This change adds the environment variables NUM_TSERVERS and NUM_SSERVERS to
the environment
with a default value of 1 if not specified in cluster.yaml and in a way
that the value is
in cluster.yaml is overridden if the environment variable already exists in
the environment
---
assemble/bin/accumulo-cluster | 18 ++++++
.../core/conf/cluster/ClusterConfigParser.java | 9 +++
.../core/conf/cluster/ClusterConfigParserTest.java | 64 +++++++++++++++++++++-
.../cluster/cluster-with-optional-services.yaml | 3 +
4 files changed, 92 insertions(+), 2 deletions(-)
diff --git a/assemble/bin/accumulo-cluster b/assemble/bin/accumulo-cluster
index 9366337d13..5bf904b091 100755
--- a/assemble/bin/accumulo-cluster
+++ b/assemble/bin/accumulo-cluster
@@ -112,6 +112,14 @@ function parse_config {
GC_HOSTS=$manager1
fi
+ if [[ -z $NUM_TSERVERS ]]; then
+ echo "INFO: ${NUM_TSERVERS} tservers will be started per host"
+ fi
+
+ # shellcheck disable=SC2153
+ if [[ -z $NUM_SSERVERS ]]; then
+ echo "INFO: ${NUM_SSERVERS} sservers will be started per host"
+ fi
}
function control_service() {
@@ -498,6 +506,16 @@ tserver:
# - localhost
#
+#
+# The following are used by the accumulo-cluster script to determine how many
servers
+# to start on each host. If the following variables are not set, then they
default to 1.
+# If the environment variable NUM_TSERVERS is set when running accumulo_cluster
+# then its value will override what is set in this file for tservers_per_host.
Likewise if
+# NUM_SSERVERS is set then it will override sservers_per_host.
+#
+tservers_per_host: 1
+sservers_per_host: 1
+
EOF
;;
restart)
diff --git
a/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
b/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
index d2c6ae7701..ec75fd0e06 100644
---
a/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
+++
b/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
@@ -76,6 +76,9 @@ public class ClusterConfigParser {
@SuppressWarnings("unchecked")
Map<String,Object> map = (Map<String,Object>) value;
map.forEach((k, v) -> flatten(parent + key, k, v, results));
+ } else if (value instanceof Number) {
+ results.put(parent + key, value.toString());
+ return;
} else {
throw new RuntimeException("Unhandled object type: " + value.getClass());
}
@@ -123,6 +126,12 @@ public class ClusterConfigParser {
config.get(sserverPrefix + ssg)));
}
+ String numTservers = config.getOrDefault("tservers_per_host", "1");
+ out.print("NUM_TSERVERS=\"${NUM_TSERVERS:=" + numTservers + "}\"\n");
+
+ String numSservers = config.getOrDefault("sservers_per_host", "1");
+ out.print("NUM_SSERVERS=\"${NUM_SSERVERS:=" + numSservers + "}\"\n");
+
out.flush();
}
diff --git
a/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
b/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
index ef0a8c92cc..8ea7e5a49f 100644
---
a/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
@@ -63,6 +63,8 @@ public class ClusterConfigParserTest {
assertFalse(contents.containsKey("compaction.compactor.queue"));
assertFalse(contents.containsKey("compaction.compactor.q1"));
assertFalse(contents.containsKey("compaction.compactor.q2"));
+ assertFalse(contents.containsKey("tservers_per_host"));
+ assertFalse(contents.containsKey("sservers_per_host"));
}
@Test
@@ -74,7 +76,7 @@ public class ClusterConfigParserTest {
Map<String,String> contents =
ClusterConfigParser.parseConfiguration(new
File(configFile.toURI()).getAbsolutePath());
- assertEquals(10, contents.size());
+ assertEquals(12, contents.size());
assertTrue(contents.containsKey("manager"));
assertEquals("localhost1 localhost2", contents.get("manager"));
assertTrue(contents.containsKey("monitor"));
@@ -98,6 +100,10 @@ public class ClusterConfigParserTest {
assertEquals("hmvm1 hmvm2 hmvm3", contents.get("sserver.highmem"));
assertTrue(contents.containsKey("sserver.cheap"));
assertEquals("burstyvm1 burstyvm2", contents.get("sserver.cheap"));
+ assertTrue(contents.containsKey("tservers_per_host"));
+ assertEquals("2", contents.get("tservers_per_host"));
+ assertTrue(contents.containsKey("sservers_per_host"));
+ assertEquals("1", contents.get("sservers_per_host"));
}
@Test
@@ -119,6 +125,58 @@ public class ClusterConfigParserTest {
PrintStream ps = new PrintStream(f);
+ URL configFile = ClusterConfigParserTest.class
+ .getResource("/org/apache/accumulo/core/conf/cluster/cluster.yaml");
+ assertNotNull(configFile);
+
+ Map<String,String> contents =
+ ClusterConfigParser.parseConfiguration(new
File(configFile.toURI()).getAbsolutePath());
+
+ ClusterConfigParser.outputShellVariables(contents, ps);
+ ps.close();
+
+ Map<String,String> expected = new HashMap<>();
+ expected.put("MANAGER_HOSTS", "localhost1 localhost2");
+ expected.put("MONITOR_HOSTS", "localhost1 localhost2");
+ expected.put("GC_HOSTS", "localhost");
+ expected.put("TSERVER_HOSTS", "localhost1 localhost2 localhost3
localhost4");
+ expected.put("NUM_TSERVERS", "${NUM_TSERVERS:=1}");
+ expected.put("NUM_SSERVERS", "${NUM_SSERVERS:=1}");
+
+ expected.replaceAll((k, v) -> {
+ return '"' + v + '"';
+ });
+
+ Map<String,String> actual = new HashMap<>();
+ try (BufferedReader rdr = Files.newBufferedReader(Paths.get(f.toURI()))) {
+ rdr.lines().forEach(l -> {
+ String[] props = l.split("=", 2);
+ actual.put(props[0], props[1]);
+ });
+ }
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testShellOutputWithOptionalComponents() throws Exception {
+
+ String userDir = System.getProperty("user.dir");
+ String targetDir = "target";
+ File dir = new File(userDir, targetDir);
+ if (!dir.exists()) {
+ if (!dir.mkdirs()) {
+ fail("Unable to make directory ${user.dir}/target");
+ }
+ }
+ File f = new File(dir,
"ClusterConfigParserTest_testShellOutputWithOptionalComponents");
+ if (!f.createNewFile()) {
+ fail("Unable to create file in ${user.dir}/target");
+ }
+ f.deleteOnExit();
+
+ PrintStream ps = new PrintStream(f);
+
URL configFile = ClusterConfigParserTest.class
.getResource("/org/apache/accumulo/core/conf/cluster/cluster-with-optional-services.yaml");
assertNotNull(configFile);
@@ -142,6 +200,8 @@ public class ClusterConfigParserTest {
expected.put("SSERVER_HOSTS_default", "localhost1 localhost2");
expected.put("SSERVER_HOSTS_highmem", "hmvm1 hmvm2 hmvm3");
expected.put("SSERVER_HOSTS_cheap", "burstyvm1 burstyvm2");
+ expected.put("NUM_TSERVERS", "${NUM_TSERVERS:=2}");
+ expected.put("NUM_SSERVERS", "${NUM_SSERVERS:=1}");
expected.replaceAll((k, v) -> {
return '"' + v + '"';
@@ -150,7 +210,7 @@ public class ClusterConfigParserTest {
Map<String,String> actual = new HashMap<>();
try (BufferedReader rdr = Files.newBufferedReader(Paths.get(f.toURI()))) {
rdr.lines().forEach(l -> {
- String[] props = l.split("=");
+ String[] props = l.split("=", 2);
actual.put(props[0], props[1]);
});
}
diff --git
a/core/src/test/resources/org/apache/accumulo/core/conf/cluster/cluster-with-optional-services.yaml
b/core/src/test/resources/org/apache/accumulo/core/conf/cluster/cluster-with-optional-services.yaml
index 57b5d53959..950cfa8ee5 100644
---
a/core/src/test/resources/org/apache/accumulo/core/conf/cluster/cluster-with-optional-services.yaml
+++
b/core/src/test/resources/org/apache/accumulo/core/conf/cluster/cluster-with-optional-services.yaml
@@ -57,3 +57,6 @@ compaction:
- q2:
- localhost3
- localhost4
+
+tservers_per_host: 2
+sservers_per_host: 1
