This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit b7a30e9da6a8f5611fcfdbc51217c9b3f9a16aba Merge: 9303d5c489 b6b4859f71 Author: Christopher Tubbs <[email protected]> AuthorDate: Wed Apr 5 13:43:23 2023 -0400 Merge branch '2.1' .../server/conf/ServerConfigurationFactory.java | 23 ++- .../accumulo/server/util/MetadataTableUtil.java | 4 - .../apache/accumulo/test/shell/ShellServerIT.java | 175 ++++++++++++--------- 3 files changed, 113 insertions(+), 89 deletions(-) diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/ServerConfigurationFactory.java index 8688636102,5453764091..bac43e5700 --- a/server/base/src/main/java/org/apache/accumulo/server/conf/ServerConfigurationFactory.java +++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ServerConfigurationFactory.java @@@ -102,11 -102,10 +102,11 @@@ public class ServerConfigurationFactor public TableConfiguration getTableConfiguration(TableId tableId) { return tableConfigs.computeIfAbsent(tableId, key -> { if (context.tableNodeExists(tableId)) { - context.getPropStore().registerAsListener(TablePropKey.of(context, tableId), deleteWatcher); + context.getPropStore().registerAsListener(TablePropKey.of(context, tableId), changeWatcher); var conf = new TableConfiguration(context, tableId, getNamespaceConfigurationForTable(tableId)); - ConfigCheckUtil.validate(conf); + ConfigCheckUtil.validate(conf, "table id: " + tableId.toString()); + return conf; } return null; @@@ -128,9 -127,9 +128,9 @@@ public NamespaceConfiguration getNamespaceConfiguration(NamespaceId namespaceId) { return namespaceConfigs.computeIfAbsent(namespaceId, key -> { context.getPropStore().registerAsListener(NamespacePropKey.of(context, namespaceId), - deleteWatcher); + changeWatcher); var conf = new NamespaceConfiguration(context, namespaceId, getSystemConfiguration()); - ConfigCheckUtil.validate(conf); + ConfigCheckUtil.validate(conf, "namespace id: " + namespaceId.toString()); return conf; }); } diff --cc test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java index 57426a9493,1aa4b36368..ae350cd475 --- a/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java +++ b/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java @@@ -77,7 -77,9 +76,8 @@@ import org.apache.accumulo.core.util.fo import org.apache.accumulo.harness.MiniClusterConfigurationCallback; import org.apache.accumulo.harness.SharedMiniClusterBase; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; -import org.apache.accumulo.test.compaction.TestCompactionStrategy; import org.apache.accumulo.test.functional.SlowIterator; + import org.apache.accumulo.test.util.Wait; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@@ -161,65 -167,113 +161,95 @@@ public class ShellServerIT extends Shar getCluster().createAccumuloClient(getPrincipal(), new PasswordToken(getRootPassword()))) { client.securityOperations().grantNamespacePermission(getPrincipal(), "", NamespacePermission.ALTER_NAMESPACE); + + final String tableBase = getUniqueNames(1)[0]; + final String table = tableBase + "_export_src"; + final String table2 = tableBase + "_import_tgt"; + + // exporttable / importtable + ts.exec("createtable " + table + " -evc", true); + make10(); + ts.exec("addsplits row5", true); + ts.exec("config -t " + table + " -s table.split.threshold=345M", true); + ts.exec("offline " + table, true); + File exportDir = new File(rootPath, "ShellServerIT.export"); + String exportUri = "file://" + exportDir; + String localTmp = "file://" + new File(rootPath, "ShellServerIT.tmp"); + ts.exec("exporttable -t " + table + " " + exportUri, true); + DistCp cp = new DistCp(new Configuration(false), null); + String import_ = "file://" + new File(rootPath, "ShellServerIT.import"); + ClientInfo info = ClientInfo.from(getCluster().getClientProperties()); + if (info.saslEnabled()) { + // DistCp bugs out trying to get a fs delegation token to perform the cp. Just copy it + // ourselves by hand. + FileSystem fs = getCluster().getFileSystem(); + FileSystem localFs = FileSystem.getLocal(new Configuration(false)); + + // Path on local fs to cp into + Path localTmpPath = new Path(localTmp); + localFs.mkdirs(localTmpPath); + + // Path in remote fs to importtable from + Path importDir = new Path(import_); + fs.mkdirs(importDir); + + // Implement a poor-man's DistCp + try (BufferedReader reader = + new BufferedReader(new FileReader(new File(exportDir, "distcp.txt"), UTF_8))) { + for (String line; (line = reader.readLine()) != null;) { + Path exportedFile = new Path(line); + // There isn't a cp on FileSystem?? + log.info("Copying {} to {}", line, localTmpPath); + fs.copyToLocalFile(exportedFile, localTmpPath); + Path tmpFile = new Path(localTmpPath, exportedFile.getName()); + log.info("Moving {} to the import directory {}", tmpFile, importDir); + fs.moveFromLocalFile(tmpFile, importDir); + } + } + } else { + String[] distCpArgs = {"-f", exportUri + "/distcp.txt", import_}; + assertEquals(0, cp.run(distCpArgs), "Failed to run distcp: " + Arrays.toString(distCpArgs)); + } + Thread.sleep(20); + ts.exec("importtable " + table2 + " " + import_, true); + ts.exec("config -t " + table2 + " -np", true, "345M", true); + ts.exec("getsplits -t " + table2, true, "row5", true); + ts.exec("constraint --list -t " + table2, true, "VisibilityConstraint=2", true); + ts.exec("online " + table, true); + ts.exec("deletetable -f " + table, true); + ts.exec("deletetable -f " + table2, true); } + } - final String table = getUniqueNames(1)[0]; - final String table2 = table + "2"; + @Test + public void propStressTest() throws Exception { + try (AccumuloClient client = + getCluster().createAccumuloClient(getPrincipal(), new PasswordToken(getRootPassword()))) { + client.securityOperations().grantNamespacePermission(getPrincipal(), "", + NamespacePermission.ALTER_NAMESPACE); - // exporttable / importtable - ts.exec("createtable " + table + " -evc", true); - make10(); - ts.exec("addsplits row5", true); - ts.exec("config -t " + table + " -s table.split.threshold=345M", true); - ts.exec("offline " + table, true); - File exportDir = new File(rootPath, "ShellServerIT.export"); - String exportUri = "file://" + exportDir; - String localTmp = "file://" + new File(rootPath, "ShellServerIT.tmp"); - ts.exec("exporttable -t " + table + " " + exportUri, true); - DistCp cp = new DistCp(new Configuration(false), null); - String import_ = "file://" + new File(rootPath, "ShellServerIT.import"); - ClientInfo info = ClientInfo.from(getCluster().getClientProperties()); - if (info.saslEnabled()) { - // DistCp bugs out trying to get a fs delegation token to perform the cp. Just copy it - // ourselves by hand. - FileSystem fs = getCluster().getFileSystem(); - FileSystem localFs = FileSystem.getLocal(new Configuration(false)); - - // Path on local fs to cp into - Path localTmpPath = new Path(localTmp); - localFs.mkdirs(localTmpPath); - - // Path in remote fs to importtable from - Path importDir = new Path(import_); - fs.mkdirs(importDir); - - // Implement a poor-man's DistCp - try (BufferedReader reader = - new BufferedReader(new FileReader(new File(exportDir, "distcp.txt"), UTF_8))) { - for (String line; (line = reader.readLine()) != null;) { - Path exportedFile = new Path(line); - // There isn't a cp on FileSystem?? - log.info("Copying {} to {}", line, localTmpPath); - fs.copyToLocalFile(exportedFile, localTmpPath); - Path tmpFile = new Path(localTmpPath, exportedFile.getName()); - log.info("Moving {} to the import directory {}", tmpFile, importDir); - fs.moveFromLocalFile(tmpFile, importDir); - } + final String table = getUniqueNames(1)[0]; + + ts.exec("createtable " + table + " -evc", true); + make10(); + ts.exec("addsplits row5", true); + + ts.exec("config -t " + table + " -s table.split.threshold=345M", true); + for (int i = 0; i < 50; i++) { + String expected = (100 + i) + "M"; + ts.exec("config -t " + table + " -s table.split.threshold=" + expected, true); + ts.exec("config -t " + table + " -np -f table.split.threshold", true, expected, true); + + ts.exec("config -t " + table + " -s table.scan.max.memory=" + expected, true); + ts.exec("config -t " + table + " -np -f table.scan.max.memory", true, expected, true); + + String bExpected = ((i % 2) == 0) ? "true" : "false"; + ts.exec("config -t " + table + " -s table.bloom.enabled=" + bExpected, true); + ts.exec("config -t " + table + " -np -f table.bloom.enabled", true, bExpected, true); } - } else { - String[] distCpArgs = {"-f", exportUri + "/distcp.txt", import_}; - assertEquals(0, cp.run(distCpArgs), "Failed to run distcp: " + Arrays.toString(distCpArgs)); } - ts.exec("importtable " + table2 + " " + import_, true); - Thread.sleep(100); - ts.exec("config -t " + table2 + " -np", true, "345M", true); - ts.exec("getsplits -t " + table2, true, "row5", true); - ts.exec("constraint --list -t " + table2, true, "VisibilityConstraint=2", true); - ts.exec("online " + table, true); - ts.exec("deletetable -f " + table, true); - ts.exec("deletetable -f " + table2, true); } - @Test - public void setscaniterDeletescaniter() throws Exception { - final String table = getUniqueNames(1)[0]; - - // setscaniter, deletescaniter - ts.exec("createtable " + table); - ts.exec("insert a cf cq 1"); - ts.exec("insert a cf cq 1"); - ts.exec("insert a cf cq 1"); - ts.input.set("true\n\n\n\nSTRING"); - ts.exec("setscaniter -class " + SUMMING_COMBINER_ITERATOR + " -p 10 -n name", true); - ts.exec("scan", true, "3", true); - ts.exec("deletescaniter -n name", true); - ts.exec("scan", true, "1", true); - ts.exec("deletetable -f " + table); - - } - @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "path provided by test") @Test public void execfile() throws Exception {
