This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 8b8a6f7 Refactor Shell ITs (#2503)
8b8a6f7 is described below
commit 8b8a6f7339fdcfbf4118c6164eb47035d857f0d9
Author: Mike Miller <mmil...@apache.org>
AuthorDate: Fri Feb 18 04:49:10 2022 -0500
Refactor Shell ITs (#2503)
* Create ShellCreateTableIT from tests moved from ShellServerIT to help
with debugging
* Create MockShell class
* Modify ShellServerIT to not delete tables after ever test
* Move ShellConfigIT into shell package
* Move ShellIT into shell package
---
.../org/apache/accumulo/test/VolumeChooserIT.java | 2 +-
.../accumulo/test/shell/ErrorMessageCallback.java | 48 ++
.../org/apache/accumulo/test/shell/MockShell.java | 185 +++++
.../accumulo/test/{ => shell}/ShellConfigIT.java | 10 +-
.../accumulo/test/shell/ShellCreateTableIT.java | 713 ++++++++++++++++++
.../apache/accumulo/test/{ => shell}/ShellIT.java | 2 +-
.../accumulo/test/{ => shell}/ShellServerIT.java | 827 +--------------------
7 files changed, 957 insertions(+), 830 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
b/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
index bb474ea..6419c6c 100644
--- a/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
@@ -65,7 +65,7 @@ public class VolumeChooserIT extends ConfigurableMacBase {
private static final String TP =
Property.TABLE_ARBITRARY_PROP_PREFIX.getKey();
static final String PREFERRED_CHOOSER_PROP = TP + "volume.preferred";
- static final String PERTABLE_CHOOSER_PROP = TP + "volume.chooser";
+ public static final String PERTABLE_CHOOSER_PROP = TP + "volume.chooser";
private static final String GP =
Property.GENERAL_ARBITRARY_PROP_PREFIX.getKey();
diff --git
a/test/src/main/java/org/apache/accumulo/test/shell/ErrorMessageCallback.java
b/test/src/main/java/org/apache/accumulo/test/shell/ErrorMessageCallback.java
new file mode 100644
index 0000000..98ac0a1
--- /dev/null
+++
b/test/src/main/java/org/apache/accumulo/test/shell/ErrorMessageCallback.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.test.shell;
+
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+
+public class ErrorMessageCallback {
+ private final String errorMessage;
+
+ public ErrorMessageCallback(Properties properties) {
+ errorMessage = checkAuths(properties);
+ }
+
+ public ErrorMessageCallback() {
+ errorMessage = "";
+ }
+
+ private String checkAuths(Properties properties) {
+ try (AccumuloClient c = Accumulo.newClient().from(properties).build()) {
+ return "Current auths for root are: " +
c.securityOperations().getUserAuthorizations("root");
+ } catch (Exception e) {
+ return "Could not check authorizations";
+ }
+ }
+
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+}
diff --git a/test/src/main/java/org/apache/accumulo/test/shell/MockShell.java
b/test/src/main/java/org/apache/accumulo/test/shell/MockShell.java
new file mode 100644
index 0000000..b7ae57e
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/shell/MockShell.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.test.shell;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.accumulo.core.clientImpl.ClientInfo;
+import org.apache.accumulo.shell.Shell;
+import org.jline.reader.LineReader;
+import org.jline.reader.LineReaderBuilder;
+import org.jline.terminal.Size;
+import org.jline.terminal.Terminal;
+import org.jline.terminal.impl.DumbTerminal;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MockShell {
+ private static final Logger shellLog =
LoggerFactory.getLogger(MockShell.class);
+ private static final ErrorMessageCallback noop = new ErrorMessageCallback();
+
+ public TestOutputStream output;
+ public StringInputStream input;
+ public Shell shell;
+ public LineReader reader;
+ public Terminal terminal;
+
+ MockShell(String user, String rootPass, String instanceName, String
zookeepers, File configFile)
+ throws IOException {
+ ClientInfo info = ClientInfo.from(configFile.toPath());
+ // start the shell
+ output = new TestOutputStream();
+ input = new StringInputStream();
+ terminal = new DumbTerminal(input, output);
+ terminal.setSize(new Size(80, 24));
+ reader = LineReaderBuilder.builder().terminal(terminal).build();
+ shell = new Shell(reader);
+ shell.setLogErrorsToConsole();
+ if (info.saslEnabled()) {
+ // Pull the kerberos principal out when we're using SASL
+ shell.config("-u", user, "-z", instanceName, zookeepers, "--config-file",
+ configFile.getAbsolutePath());
+ } else {
+ shell.config("-u", user, "-p", rootPass, "-z", instanceName, zookeepers,
"--config-file",
+ configFile.getAbsolutePath());
+ }
+ exec("quit", true);
+ shell.start();
+ shell.setExit(false);
+ }
+
+ String exec(String cmd) throws IOException {
+ output.clear();
+ shell.execCommand(cmd, true, true);
+ return output.get();
+ }
+
+ String exec(String cmd, boolean expectGoodExit) throws IOException {
+ return exec(cmd, expectGoodExit, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, ErrorMessageCallback
callback)
+ throws IOException {
+ String result = exec(cmd);
+ if (expectGoodExit)
+ assertGoodExit("", true, callback);
+ else
+ assertBadExit("", true, callback);
+ return result;
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString) throws
IOException {
+ return exec(cmd, expectGoodExit, expectString, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString,
+ ErrorMessageCallback callback) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, true, callback);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, boolean
stringPresent)
+ throws IOException {
+ return exec(cmd, expectGoodExit, expectString, stringPresent, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, boolean
stringPresent,
+ ErrorMessageCallback callback) throws IOException {
+ String result = exec(cmd);
+ if (expectGoodExit)
+ assertGoodExit(expectString, stringPresent, callback);
+ else
+ assertBadExit(expectString, stringPresent, callback);
+ return result;
+ }
+
+ void assertGoodExit(String s, boolean stringPresent) {
+ assertGoodExit(s, stringPresent, noop);
+ }
+
+ void assertGoodExit(String s, boolean stringPresent, ErrorMessageCallback
callback) {
+ shellLog.debug("Shell Output: '{}'", output.get());
+ if (shell.getExitCode() != 0) {
+ String errorMsg = callback.getErrorMessage();
+ assertEquals(errorMsg, 0, shell.getExitCode());
+ }
+
+ if (!s.isEmpty())
+ assertEquals(s + " present in " + output.get() + " was not " +
stringPresent, stringPresent,
+ output.get().contains(s));
+ }
+
+ void assertBadExit(String s, boolean stringPresent, ErrorMessageCallback
callback) {
+ shellLog.debug(output.get());
+ if (shell.getExitCode() == 0) {
+ String errorMsg = callback.getErrorMessage();
+ assertTrue(errorMsg, shell.getExitCode() > 0);
+ }
+
+ if (!s.isEmpty())
+ assertEquals(s + " present in " + output.get() + " was not " +
stringPresent, stringPresent,
+ output.get().contains(s));
+ shell.resetExitCode();
+ }
+
+ void writeToHistory(String cmd) {
+ input.set(cmd);
+ reader.readLine();
+ }
+
+ static class TestOutputStream extends OutputStream {
+ StringBuilder sb = new StringBuilder();
+
+ @Override
+ public void write(int b) {
+ sb.append((char) (0xff & b));
+ }
+
+ public String get() {
+ return sb.toString();
+ }
+
+ public void clear() {
+ sb.setLength(0);
+ }
+ }
+
+ static class StringInputStream extends InputStream {
+ private String source = "";
+ private int offset = 0;
+
+ @Override
+ public int read() {
+ if (offset == source.length())
+ return '\n';
+ else
+ return source.charAt(offset++);
+ }
+
+ public void set(String other) {
+ source = other;
+ offset = 0;
+ }
+ }
+}
diff --git a/test/src/main/java/org/apache/accumulo/test/ShellConfigIT.java
b/test/src/main/java/org/apache/accumulo/test/shell/ShellConfigIT.java
similarity index 94%
rename from test/src/main/java/org/apache/accumulo/test/ShellConfigIT.java
rename to test/src/main/java/org/apache/accumulo/test/shell/ShellConfigIT.java
index 308bbbd..f7b1667 100644
--- a/test/src/main/java/org/apache/accumulo/test/ShellConfigIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/shell/ShellConfigIT.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.accumulo.test;
+package org.apache.accumulo.test.shell;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.accumulo.test.VolumeChooserIT.PERTABLE_CHOOSER_PROP;
@@ -36,7 +36,7 @@ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.harness.AccumuloClusterHarness;
import org.apache.accumulo.harness.conf.StandaloneAccumuloClusterConfiguration;
import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl;
-import org.apache.accumulo.test.ShellServerIT.TestShell;
+import org.apache.accumulo.test.FairVolumeChooser;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -95,13 +95,13 @@ public class ShellConfigIT extends AccumuloClusterHarness {
assertNotNull(clientPropsFile);
- TestShell ts = null;
+ MockShell ts = null;
if (token instanceof PasswordToken) {
String passwd = new String(((PasswordToken) token).getPassword(), UTF_8);
- ts = new TestShell(getAdminPrincipal(), passwd,
getCluster().getInstanceName(),
+ ts = new MockShell(getAdminPrincipal(), passwd,
getCluster().getInstanceName(),
getCluster().getZooKeepers(), clientPropsFile);
} else if (token instanceof KerberosToken) {
- ts = new TestShell(getAdminPrincipal(), null,
getCluster().getInstanceName(),
+ ts = new MockShell(getAdminPrincipal(), null,
getCluster().getInstanceName(),
getCluster().getZooKeepers(), clientPropsFile);
} else {
fail("Unknown token type");
diff --git
a/test/src/main/java/org/apache/accumulo/test/shell/ShellCreateTableIT.java
b/test/src/main/java/org/apache/accumulo/test/shell/ShellCreateTableIT.java
new file mode 100644
index 0000000..2499b16
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/shell/ShellCreateTableIT.java
@@ -0,0 +1,713 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.test.shell;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.Files.newBufferedReader;
+import static java.util.Objects.requireNonNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.UUID;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.harness.MiniClusterConfigurationCallback;
+import org.apache.accumulo.harness.SharedMiniClusterBase;
+import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.categories.MiniClusterOnlyTests;
+import org.apache.accumulo.test.categories.SunnyDayTests;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({MiniClusterOnlyTests.class, SunnyDayTests.class})
+public class ShellCreateTableIT extends SharedMiniClusterBase {
+
+ private MockShell ts;
+
+ private static class SSCTITCallback implements
MiniClusterConfigurationCallback {
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration
coreSite) {
+ // Only one tserver to avoid race conditions on ZK propagation (auths
and configuration)
+ cfg.setNumTservers(1);
+ // Set the min span to 0 so we will definitely get all the traces back.
See ACCUMULO-4365
+ Map<String,String> siteConf = cfg.getSiteConfig();
+ cfg.setSiteConfig(siteConf);
+ }
+ }
+
+ @BeforeClass
+ public static void setupMiniCluster() throws Exception {
+ SharedMiniClusterBase.startMiniClusterWithConfig(new SSCTITCallback());
+ }
+
+ @Before
+ public void setupShell() throws Exception {
+ ts = new MockShell(getPrincipal(), getRootPassword(),
+ getCluster().getConfig().getInstanceName(),
getCluster().getConfig().getZooKeepers(),
+ getCluster().getConfig().getClientPropsFile());
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() {
+ SharedMiniClusterBase.stopMiniCluster();
+ }
+
+ @After
+ public void tearDownShell() {
+ ts.shell.shutdown();
+ }
+
+ @Test
+ public void testCreateTableWithLocalityGroups() throws Exception {
+ final String table = getUniqueNames(1)[0];
+ ts.exec("createtable " + table + " -l locg1=fam1,fam2", true);
+ try (AccumuloClient accumuloClient =
Accumulo.newClient().from(getClientProps()).build()) {
+ Map<String,Set<Text>> lMap =
accumuloClient.tableOperations().getLocalityGroups(table);
+ Set<Text> expectedColFams = Set.of(new Text("fam1"), new Text("fam2"));
+ for (Map.Entry<String,Set<Text>> entry : lMap.entrySet()) {
+ assertEquals("locg1", entry.getKey());
+ assertTrue(entry.getValue().containsAll(expectedColFams));
+ }
+ ts.exec("deletetable -f " + table);
+ }
+ }
+
+ /**
+ * Due to the existing complexity of the createtable command, the
createtable help only displays
+ * an example of setting one locality group. It is possible to set multiple
groups if needed. This
+ * test verifies that capability.
+ */
+ @Test
+ public void testCreateTableWithMultipleLocalityGroups() throws Exception {
+ final String table = getUniqueNames(1)[0];
+ ts.exec("createtable " + table + " -l locg1=fam1,fam2 locg2=colfam1",
true);
+ try (AccumuloClient accumuloClient =
Accumulo.newClient().from(getClientProps()).build()) {
+ Map<String,Set<Text>> lMap =
accumuloClient.tableOperations().getLocalityGroups(table);
+ assertTrue(lMap.containsKey("locg1"));
+ assertTrue(lMap.containsKey("locg2"));
+ Set<Text> expectedColFams1 = Set.of(new Text("fam1"), new Text("fam2"));
+ Set<Text> expectedColFams2 = Set.of(new Text("colfam1"));
+ assertTrue(lMap.get("locg1").containsAll(expectedColFams1));
+ assertTrue(lMap.get("locg2").containsAll(expectedColFams2));
+ ts.exec("deletetable -f " + table);
+ }
+ }
+
+ @Test
+ public void testCreateTableWithLocalityGroupsBadArguments() throws
IOException {
+ final String table = getUniqueNames(1)[0];
+ ts.exec("createtable " + table + " -l locg1 fam1,fam2", false);
+ ts.exec("createtable " + table + "-l", false);
+ ts.exec("createtable " + table + " -l locg1 = fam1,fam2", false);
+ ts.exec("createtable " + table + " -l locg1=fam1 ,fam2", false);
+ ts.exec("createtable " + table + " -l locg1=fam1,fam2 locg1=fam3,fam4",
false);
+ ts.exec("createtable " + table + " -l locg1=fam1,fam2 locg2=fam1", false);
+ ts.exec("createtable " + table + " -l locg1", false);
+ ts.exec("createtable " + table + " group=fam1", false);
+ ts.exec("createtable " + table + "-l fam1,fam2", false);
+ }
+
+ @Test
+ public void testCreateTableWithIterators() throws Exception {
+ final String tmpTable = "tmpTable";
+ final String table = getUniqueNames(1)[0];
+
+ // create iterator profile
+ // Will use tmpTable for creating profile since setshelliter is requiring
a table
+ // even though its command line help indicates that it is optional. Likely
due to
+ // the fact that setshelliter extends setiter, which does require a table
argument.
+ ts.exec("createtable " + tmpTable, true);
+ String output = ts.exec("tables");
+ assertTrue(output.contains(tmpTable));
+
+ ts.input.set("\n5000\n\n");
+ ts.exec("setshelliter -n itname -p 10 -pn profile1 -ageoff", true);
+ output = ts.exec("listshelliter");
+ assertTrue(output.contains("Profile : profile1"));
+
+ // create table making use of the iterator profile
+ ts.exec("createtable " + table + " -i profile1:scan,minc", true);
+ ts.exec("insert foo a b c", true);
+ ts.exec("scan", true, "foo a:b []\tc");
+ ts.exec("sleep 6", true);
+ ts.exec("scan", true, "", true);
+ ts.exec("deletetable -f " + table);
+ ts.exec("deletetable -f " + tmpTable);
+ }
+
+ /**
+ * Due to the existing complexity of the createtable command, the
createtable help only displays
+ * an example of setting one iterator upon table creation. It is possible to
set multiple if
+ * needed. This test verifies that capability.
+ */
+ @Test
+ public void testCreateTableWithMultipleIterators() throws Exception {
+ final String tmpTable = "tmpTable";
+ final String table = getUniqueNames(1)[0];
+
+ // create iterator profile
+ // Will use tmpTable for creating profile since setshelliter is requiring
a table
+ // even though its command line help indicates that it is optional. Likely
due to
+ // the fact that setshelliter extends setiter, which does require a table
argument.
+ ts.exec("createtable " + tmpTable, true);
+ String output = ts.exec("tables");
+ assertTrue(output.contains(tmpTable));
+
+ ts.input.set("\n5000\n\n");
+ ts.exec("setshelliter -n itname -p 10 -pn profile1 -ageoff", true);
+ output = ts.exec("listshelliter");
+ assertTrue(output.contains("Profile : profile1"));
+
+ ts.input.set("2\n");
+ ts.exec("setshelliter -n iter2 -p 11 -pn profile2 -vers", true);
+ output = ts.exec("listshelliter");
+ assertTrue(output.contains("Profile : profile2"));
+
+ // create table making use of the iterator profiles
+ ts.exec("createtable " + table + " -i profile1:scan,minc profile2:all ",
true);
+ ts.exec("insert foo a b c", true);
+ ts.exec("scan", true, "foo a:b []\tc");
+ ts.exec("sleep 6", true);
+ ts.exec("scan", true, "", true);
+ output = ts.exec("listiter -t " + table + " -all");
+ assertTrue(output.contains("Iterator itname, scan scope options"));
+ assertTrue(output.contains("Iterator itname, minc scope options"));
+ assertFalse(output.contains("Iterator itname, majc scope options"));
+ assertTrue(output.contains("Iterator iter2, scan scope options"));
+ assertTrue(output.contains("Iterator iter2, minc scope options"));
+ assertTrue(output.contains("Iterator iter2, majc scope options"));
+ ts.exec("deletetable -f " + table);
+ ts.exec("deletetable -f " + tmpTable);
+ }
+
+ @Test
+ public void testCreateTableWithIteratorsBadArguments() throws IOException {
+ final String tmpTable = "tmpTable";
+ final String table = getUniqueNames(1)[0];
+ ts.exec("createtable " + tmpTable, true);
+ String output = ts.exec("tables");
+ assertTrue(output.contains(tmpTable));
+ ts.input.set("\n5000\n\n");
+ ts.exec("setshelliter -n itname -p 10 -pn profile1 -ageoff", true);
+ output = ts.exec("listshelliter");
+ assertTrue(output.contains("Profile : profile1"));
+ // test various bad argument calls
+ ts.exec("createtable " + table + " -i noprofile:scan,minc", false);
+ ts.exec("createtable " + table + " -i profile1:scan,minc,all,majc", false);
+ ts.exec("createtable " + table + " -i profile1:scan,all,majc", false);
+ ts.exec("createtable " + table + " -i profile1:scan,min,majc", false);
+ ts.exec("createtable " + table + " -i profile1:scan,max,all", false);
+ ts.exec("createtable " + table + " -i profile1:", false);
+ ts.exec("createtable " + table + " -i profile1: ", false);
+ ts.exec("createtable " + table + " -i profile1:-scan", false);
+ ts.exec("createtable " + table + " profile1:majc", false);
+ ts.exec("createtable " + table + " -i profile1: all", false);
+ ts.exec("createtable " + table + " -i profile1: All", false);
+ ts.exec("createtable " + table + " -i profile1: scan", false);
+ ts.exec("createtable " + table + " -i profile1:minc scan", false);
+ ts.exec("createtable " + table + " -i profile1:minc,Scan", false);
+ ts.exec("createtable " + table + " -i profile1:minc, scan", false);
+ ts.exec("createtable " + table + " -i profile1:minc,,scan", false);
+ ts.exec("createtable " + table + " -i profile1:minc,minc", false);
+ ts.exec("createtable " + table + " -i profile1:minc,Minc", false);
+ ts.exec("createtable " + table + " -i profile1:minc, ,scan", false);
+ ts.exec("createtable " + table + "-i", false);
+ ts.exec("createtable " + table + "-i ", false);
+ ts.exec("deletetable -f " + tmpTable);
+ }
+
+ /**
+ * Verify that table can be created in offline status and then be brought
online.
+ */
+ @Test
+ public void testCreateTableOffline() throws IOException {
+ final String tableName = getUniqueNames(1)[0];
+ ts.exec("createtable " + tableName + " -o", true);
+ String output = ts.exec("tables");
+ assertTrue(output.contains(tableName));
+ output = ts.exec("scan -t " + tableName, false, "is offline", true);
+ assertTrue(output.contains("TableOfflineException"));
+ ts.exec("table " + tableName, true);
+ ts.exec("online", true);
+ ts.exec("scan", true);
+ ts.exec("deletetable -f " + tableName, true);
+ }
+
+ /**
+ * Use shell to create a table with a supplied file containing splits.
+ *
+ * The splits will be contained in a file, sorted and un-encoded with no
repeats or blank lines.
+ */
+ @Test
+ public void testCreateTableWithSplitsFile1()
+ throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
+ String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ generateSplitsFile(splitsFile, 1000, 12, false, false, true, false,
false);
+ SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
+ final String tableName = getUniqueNames(1)[0];
+ ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
+ Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
+ assertEquals(expectedSplits, new TreeSet<>(createdSplits));
+ } finally {
+ Files.delete(Paths.get(splitsFile));
+ }
+ }
+
+ /**
+ * Use shell to create a table with a supplied file containing splits.
+ *
+ * The splits will be contained in a file, unsorted and un-encoded with no
repeats or blank lines.
+ */
+ @Test
+ public void testCreateTableWithSplitsFile2()
+ throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
+ String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ generateSplitsFile(splitsFile, 300, 12, false, false, false, false,
false);
+ SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
+ final String tableName = getUniqueNames(1)[0];
+ ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
+ Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
+ assertEquals(expectedSplits, new TreeSet<>(createdSplits));
+ } finally {
+ Files.delete(Paths.get(splitsFile));
+ }
+ }
+
+ /**
+ * Use shell to create a table with a supplied file containing splits.
+ *
+ * The splits will be contained in a file, sorted and encoded with no
repeats or blank lines.
+ */
+ @Test
+ public void testCreateTableWithSplitsFile3()
+ throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
+ String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ generateSplitsFile(splitsFile, 100, 23, false, true, true, false, false);
+ SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
+ final String tableName = getUniqueNames(1)[0];
+ ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
+ Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
+ assertEquals(expectedSplits, new TreeSet<>(createdSplits));
+ } finally {
+ Files.delete(Paths.get(splitsFile));
+ }
+ }
+
+ /**
+ * Use shell to create a table with a supplied file containing splits.
+ *
+ * The splits will be contained in a file, sorted and un-encoded with a
blank line and no repeats.
+ */
+ @Test
+ public void testCreateTableWithSplitsFile4()
+ throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
+ String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ generateSplitsFile(splitsFile, 100, 31, false, false, true, true, false);
+ SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
+ final String tableName = getUniqueNames(1)[0];
+ ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
+ Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
+ assertEquals(expectedSplits, new TreeSet<>(createdSplits));
+ } finally {
+ Files.delete(Paths.get(splitsFile));
+ }
+ }
+
+ /**
+ * Use shell to create a table with a supplied file containing splits.
+ *
+ * The splits will be contained in a file, sorted and un-encoded with a
blank line and no repeats.
+ */
+ @Test
+ public void testCreateTableWithSplitsFile5()
+ throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
+ String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ generateSplitsFile(splitsFile, 100, 32, false, false, true, false, true);
+ SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
+ final String tableName = getUniqueNames(1)[0];
+ ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
+ Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
+ assertEquals(expectedSplits, new TreeSet<>(createdSplits));
+ } finally {
+ Files.delete(Paths.get(splitsFile));
+ }
+ }
+
+ /**
+ * Use shell to create a table with a supplied file containing splits.
+ *
+ * The splits will be contained in a file, unsorted and un-encoded with a
blank line and repeats.
+ */
+ @Test
+ public void testCreateTableWithSplitsFile6()
+ throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
+ String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ generateSplitsFile(splitsFile, 100, 12, false, false, false, true, true);
+ SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
+ final String tableName = getUniqueNames(1)[0];
+ ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
+ Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
+ assertEquals(expectedSplits, new TreeSet<>(createdSplits));
+ } finally {
+ Files.delete(Paths.get(splitsFile));
+ }
+ }
+
+ /**
+ * Use shell to create a table with a supplied file containing splits.
+ *
+ * The splits will be contained in a file, sorted and encoded with a blank
line and repeats.
+ */
+ @Test
+ public void testCreateTableWithSplitsFile7()
+ throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
+ String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ generateSplitsFile(splitsFile, 100, 12, false, false, true, true, true);
+ SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
+ final String tableName = getUniqueNames(1)[0];
+ ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
+ Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
+ assertEquals(expectedSplits, new TreeSet<>(createdSplits));
+ } finally {
+ Files.delete(Paths.get(splitsFile));
+ }
+ }
+
+ /**
+ * Use shell to create a table with a supplied file containing splits.
+ *
+ * The splits file will be empty.
+ */
+ @Test
+ public void testCreateTableWithEmptySplitFile()
+ throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
+ String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ generateSplitsFile(splitsFile, 0, 0, false, false, false, false, false);
+ final String tableName = getUniqueNames(1)[0];
+ ts.exec("createtable " + tableName + " -sf " + splitsFile, false);
+ assertThrows(TableNotFoundException.class,
+ () -> client.tableOperations().listSplits(tableName));
+ } finally {
+ Files.delete(Paths.get(splitsFile));
+ }
+ }
+
+ /**
+ * Use shell to create a table that used splits from another table.
+ */
+ @Test
+ public void testCreateTableWithCopySplitsFromOtherTable()
+ throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
+ // create a table and add some splits
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ final String[] tableNames = getUniqueNames(2);
+ final String tableName0 = tableNames[0];
+ final String tableName2 = tableNames[1];
+
+ ts.exec("createtable " + tableName0, true);
+ String output = ts.exec("tables", true);
+ assertTrue(output.contains(tableName0));
+ ts.exec("table " + tableName0, true);
+ // add splits to this table using the addsplits command.
+ List<Text> splits = new ArrayList<>();
+ splits.add(new Text("ccccc"));
+ splits.add(new Text("fffff"));
+ splits.add(new Text("mmmmm"));
+ splits.add(new Text("sssss"));
+ ts.exec("addsplits " + splits.get(0) + " " + splits.get(1) + " " +
splits.get(2) + " "
+ + splits.get(3), true);
+ // Now create a table that will used the previous tables splits and
create them at table
+ // creation
+ ts.exec("createtable " + tableName2 + " --copy-splits " + tableName0,
true);
+ ts.exec("table " + tableName0, true);
+ String tablesOutput = ts.exec("tables", true);
+ assertTrue(tablesOutput.contains(tableName2));
+ Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName2);
+ assertEquals(new TreeSet<>(splits), new TreeSet<>(createdSplits));
+ ts.exec("deletetable -f " + tableName0, true);
+ ts.exec("deletetable -f " + tableName2, true);
+ }
+ }
+
+ /**
+ * Use shell to create a table with a supplied file containing splits.
+ *
+ * The splits will be contained in a file, sorted and encoded with no
repeats or blank lines.
+ */
+ @Test
+ public void testCreateTableWithBinarySplitsFile1()
+ throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
+ String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ generateSplitsFile(splitsFile, 200, 12, true, true, true, false, false);
+ SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
+ final String tableName = getUniqueNames(1)[0];
+ ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
+ Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
+ assertEquals(expectedSplits, new TreeSet<>(createdSplits));
+ } finally {
+ Files.delete(Paths.get(splitsFile));
+ }
+ }
+
+ /**
+ * Use shell to create a table with a supplied file containing splits.
+ *
+ * The splits will be contained in a file, unsorted and encoded with no
repeats or blank lines.
+ */
+ @Test
+ public void testCreateTableWithBinarySplitsFile2()
+ throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
+ String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ generateSplitsFile(splitsFile, 300, 12, true, true, false, false, false);
+ SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
+ final String tableName = getUniqueNames(1)[0];
+ ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
+ Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
+ assertEquals(expectedSplits, new TreeSet<>(createdSplits));
+ } finally {
+ Files.delete(Paths.get(splitsFile));
+ }
+ }
+
+ /**
+ * Use shell to create a table with a supplied file containing splits.
+ *
+ * The splits will be contained in a file, sorted and encoded with no
repeats or blank lines.
+ */
+ @Test
+ public void testCreateTableWithBinarySplitsFile3()
+ throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
+ String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ generateSplitsFile(splitsFile, 100, 23, true, true, true, false, false);
+ SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
+ final String tableName = getUniqueNames(1)[0];
+ ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
+ Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
+ assertEquals(expectedSplits, new TreeSet<>(createdSplits));
+ } finally {
+ Files.delete(Paths.get(splitsFile));
+ }
+ }
+
+ /**
+ * Use shell to create a table with a supplied file containing splits.
+ *
+ * The splits will be contained in a file, sorted and encoded with a blank
line and no repeats.
+ */
+ @Test
+ public void testCreateTableWithBinarySplitsFile4()
+ throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
+ String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ generateSplitsFile(splitsFile, 100, 31, true, true, true, true, false);
+ SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
+ final String tableName = getUniqueNames(1)[0];
+ ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
+ Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
+ assertEquals(expectedSplits, new TreeSet<>(createdSplits));
+ } finally {
+ Files.delete(Paths.get(splitsFile));
+ }
+ }
+
+ /**
+ * Use shell to create a table with a supplied file containing splits.
+ *
+ * The splits will be contained in a file, sorted and encoded with a blank
line and no repeats.
+ */
+ @Test
+ public void testCreateTableWithBinarySplitsFile5()
+ throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
+ String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ generateSplitsFile(splitsFile, 100, 32, true, true, true, false, true);
+ SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
+ final String tableName = getUniqueNames(1)[0];
+ ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
+ Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
+ assertEquals(expectedSplits, new TreeSet<>(createdSplits));
+ } finally {
+ Files.delete(Paths.get(splitsFile));
+ }
+ }
+
+ /**
+ * Use shell to create a table with a supplied file containing splits.
+ *
+ * The splits will be contained in a file, unsorted and encoded with a blank
line and repeats.
+ */
+ @Test
+ public void testCreateTableWithBinarySplitsFile6()
+ throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
+ String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ generateSplitsFile(splitsFile, 100, 12, true, true, false, true, true);
+ SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
+ final String tableName = getUniqueNames(1)[0];
+ ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
+ Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
+ assertEquals(expectedSplits, new TreeSet<>(createdSplits));
+ } finally {
+ Files.delete(Paths.get(splitsFile));
+ }
+ }
+
+ /**
+ * Use shell to create a table with a supplied file containing splits.
+ *
+ * The splits will be contained in a file, sorted and encoded with a blank
line and repeats.
+ */
+ @Test
+ public void testCreateTableWithBinarySplitsFile7()
+ throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
+ String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ generateSplitsFile(splitsFile, 100, 12, true, true, true, true, true);
+ SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
+ final String tableName = getUniqueNames(1)[0];
+ ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
+ Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
+ assertEquals(expectedSplits, new TreeSet<>(createdSplits));
+ } finally {
+ Files.delete(Paths.get(splitsFile));
+ }
+ }
+
+ private SortedSet<Text> readSplitsFromFile(final String splitsFile) throws
IOException {
+ SortedSet<Text> splits = new TreeSet<>();
+ try (BufferedReader reader = newBufferedReader(Paths.get(splitsFile))) {
+ String split;
+ while ((split = reader.readLine()) != null) {
+ Text unencodedString = decode(split);
+ if (unencodedString != null)
+ splits.add(unencodedString);
+ }
+ }
+ return splits;
+ }
+
+ private void generateSplitsFile(final String splitsFile, final int numItems,
final int len,
+ final boolean binarySplits, final boolean encoded, final boolean sort,
+ final boolean addBlankLine, final boolean repeat) throws IOException {
+
+ java.nio.file.Path splitsPath = java.nio.file.Paths.get(splitsFile);
+ int insertAt = (len % 2 == 0) ? len / 2 : (len + 1) / 2;
+ Collection<Text> sortedSplits = null;
+ Collection<Text> randomSplits;
+
+ if (binarySplits)
+ randomSplits = generateBinarySplits(numItems, len);
+ else
+ randomSplits = generateNonBinarySplits(numItems, len);
+
+ if (sort)
+ sortedSplits = new TreeSet<>(randomSplits);
+
+ try (BufferedWriter writer = Files.newBufferedWriter(splitsPath, UTF_8)) {
+ int cnt = 0;
+ Collection<Text> splits;
+ if (sort)
+ splits = sortedSplits;
+ else
+ splits = randomSplits;
+
+ for (Text text : splits) {
+ if (addBlankLine && cnt++ == insertAt)
+ writer.write('\n');
+ writer.write(encode(text, encoded) + '\n');
+ if (repeat)
+ writer.write(encode(text, encoded) + '\n');
+ }
+ }
+ }
+
+ private Collection<Text> generateNonBinarySplits(final int numItems, final
int len) {
+ Set<Text> splits = new HashSet<>();
+ for (int i = 0; i < numItems; i++) {
+ splits.add(getRandomText(len));
+ }
+ return splits;
+ }
+
+ private Collection<Text> generateBinarySplits(final int numItems, final int
len) {
+ Set<Text> splits = new HashSet<>();
+ for (int i = 0; i < numItems; i++) {
+ byte[] split = new byte[len];
+ random.nextBytes(split);
+ splits.add(new Text(split));
+ }
+ return splits;
+ }
+
+ private Text getRandomText(final int len) {
+ int desiredLen = Math.min(len, 32);
+ return new Text(
+ String.valueOf(UUID.randomUUID()).replaceAll("-", "").substring(0,
desiredLen - 1));
+ }
+
+ private static String encode(final Text text, final boolean encode) {
+ if (text.toString().isBlank())
+ return null;
+ return encode ?
Base64.getEncoder().encodeToString(TextUtil.getBytes(text)) : text.toString();
+ }
+
+ private Text decode(final String text) {
+ if (requireNonNull(text).isBlank())
+ return null;
+ return new Text(text);
+ }
+}
diff --git a/test/src/main/java/org/apache/accumulo/test/ShellIT.java
b/test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java
similarity index 99%
rename from test/src/main/java/org/apache/accumulo/test/ShellIT.java
rename to test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java
index 5a8b7a7..cdfa98f 100644
--- a/test/src/main/java/org/apache/accumulo/test/ShellIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.accumulo.test;
+package org.apache.accumulo.test.shell;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
diff --git a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
b/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
similarity index 70%
rename from test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
rename to test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
index 26da4af..65412dc 100644
--- a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
@@ -16,11 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.accumulo.test;
+package org.apache.accumulo.test.shell;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.nio.file.Files.newBufferedReader;
-import static java.util.Objects.requireNonNull;
import static
org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -30,41 +28,26 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.BufferedReader;
-import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.io.PrintWriter;
import java.lang.reflect.Constructor;
-import java.nio.file.Files;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Base64;
-import java.util.Collection;
import java.util.Collections;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.client.sample.RowSampler;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
@@ -84,13 +67,11 @@ import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVWriter;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.TextUtil;
import org.apache.accumulo.core.util.format.Formatter;
import org.apache.accumulo.core.util.format.FormatterConfig;
import org.apache.accumulo.harness.MiniClusterConfigurationCallback;
import org.apache.accumulo.harness.SharedMiniClusterBase;
import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.shell.Shell;
import org.apache.accumulo.test.categories.MiniClusterOnlyTests;
import org.apache.accumulo.test.categories.SunnyDayTests;
import org.apache.accumulo.test.compaction.TestCompactionStrategy;
@@ -100,11 +81,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.tools.DistCp;
-import org.jline.reader.LineReader;
-import org.jline.reader.LineReaderBuilder;
-import org.jline.terminal.Size;
import org.jline.terminal.Terminal;
-import org.jline.terminal.impl.DumbTerminal;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assume;
@@ -129,170 +106,9 @@ public class ShellServerIT extends SharedMiniClusterBase {
private static final Property VFS_CONTEXT_CLASSPATH_PROPERTY =
Property.VFS_CONTEXT_CLASSPATH_PROPERTY;
- public static class TestOutputStream extends OutputStream {
- StringBuilder sb = new StringBuilder();
-
- @Override
- public void write(int b) {
- sb.append((char) (0xff & b));
- }
-
- public String get() {
- return sb.toString();
- }
-
- public void clear() {
- sb.setLength(0);
- }
- }
-
private static final Logger log =
LoggerFactory.getLogger(ShellServerIT.class);
- public static class StringInputStream extends InputStream {
- private String source = "";
- private int offset = 0;
-
- @Override
- public int read() {
- if (offset == source.length())
- return '\n';
- else
- return source.charAt(offset++);
- }
-
- public void set(String other) {
- source = other;
- offset = 0;
- }
- }
-
- private abstract static class ErrorMessageCallback {
- public abstract String getErrorMessage();
- }
-
- private static class NoOpErrorMessageCallback extends ErrorMessageCallback {
- private static final String empty = "";
-
- @Override
- public String getErrorMessage() {
- return empty;
- }
- }
-
- public static class TestShell {
- private static final Logger shellLog =
LoggerFactory.getLogger(TestShell.class);
- public TestOutputStream output;
- public StringInputStream input;
- public Shell shell;
- public LineReader reader;
- public Terminal terminal;
-
- TestShell(String user, String rootPass, String instanceName, String
zookeepers, File configFile)
- throws IOException {
- ClientInfo info = ClientInfo.from(configFile.toPath());
- // start the shell
- output = new TestOutputStream();
- input = new StringInputStream();
- terminal = new DumbTerminal(input, output);
- terminal.setSize(new Size(80, 24));
- reader = LineReaderBuilder.builder().terminal(terminal).build();
- shell = new Shell(reader);
- shell.setLogErrorsToConsole();
- if (info.saslEnabled()) {
- // Pull the kerberos principal out when we're using SASL
- shell.config("-u", user, "-z", instanceName, zookeepers,
"--config-file",
- configFile.getAbsolutePath());
- } else {
- shell.config("-u", user, "-p", rootPass, "-z", instanceName,
zookeepers, "--config-file",
- configFile.getAbsolutePath());
- }
- exec("quit", true);
- shell.start();
- shell.setExit(false);
- }
-
- String exec(String cmd) throws IOException {
- output.clear();
- shell.execCommand(cmd, true, true);
- return output.get();
- }
-
- String exec(String cmd, boolean expectGoodExit) throws IOException {
- return exec(cmd, expectGoodExit, noop);
- }
-
- String exec(String cmd, boolean expectGoodExit, ErrorMessageCallback
callback)
- throws IOException {
- String result = exec(cmd);
- if (expectGoodExit)
- assertGoodExit("", true, callback);
- else
- assertBadExit("", true, callback);
- return result;
- }
-
- String exec(String cmd, boolean expectGoodExit, String expectString)
throws IOException {
- return exec(cmd, expectGoodExit, expectString, noop);
- }
-
- String exec(String cmd, boolean expectGoodExit, String expectString,
- ErrorMessageCallback callback) throws IOException {
- return exec(cmd, expectGoodExit, expectString, true, callback);
- }
-
- String exec(String cmd, boolean expectGoodExit, String expectString,
boolean stringPresent)
- throws IOException {
- return exec(cmd, expectGoodExit, expectString, stringPresent, noop);
- }
-
- String exec(String cmd, boolean expectGoodExit, String expectString,
boolean stringPresent,
- ErrorMessageCallback callback) throws IOException {
- String result = exec(cmd);
- if (expectGoodExit)
- assertGoodExit(expectString, stringPresent, callback);
- else
- assertBadExit(expectString, stringPresent, callback);
- return result;
- }
-
- void assertGoodExit(String s, boolean stringPresent) {
- assertGoodExit(s, stringPresent, noop);
- }
-
- void assertGoodExit(String s, boolean stringPresent, ErrorMessageCallback
callback) {
- shellLog.debug("Shell Output: '{}'", output.get());
- if (shell.getExitCode() != 0) {
- String errorMsg = callback.getErrorMessage();
- assertEquals(errorMsg, 0, shell.getExitCode());
- }
-
- if (!s.isEmpty())
- assertEquals(s + " present in " + output.get() + " was not " +
stringPresent, stringPresent,
- output.get().contains(s));
- }
-
- void assertBadExit(String s, boolean stringPresent, ErrorMessageCallback
callback) {
- shellLog.debug(output.get());
- if (shell.getExitCode() == 0) {
- String errorMsg = callback.getErrorMessage();
- assertTrue(errorMsg, shell.getExitCode() > 0);
- }
-
- if (!s.isEmpty())
- assertEquals(s + " present in " + output.get() + " was not " +
stringPresent, stringPresent,
- output.get().contains(s));
- shell.resetExitCode();
- }
-
- void writeToHistory(String cmd) {
- input.set(cmd);
- reader.readLine();
- }
- }
-
- private static final NoOpErrorMessageCallback noop = new
NoOpErrorMessageCallback();
-
- private TestShell ts;
+ private MockShell ts;
private static String rootPath;
@@ -325,7 +141,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
@Before
public void setupShell() throws Exception {
- ts = new TestShell(getPrincipal(), getRootPassword(),
+ ts = new MockShell(getPrincipal(), getRootPassword(),
getCluster().getConfig().getInstanceName(),
getCluster().getConfig().getZooKeepers(),
getCluster().getConfig().getClientPropsFile());
}
@@ -336,20 +152,6 @@ public class ShellServerIT extends SharedMiniClusterBase {
}
@After
- public void deleteTables() throws Exception {
- try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
- for (String table : c.tableOperations().list()) {
- if (!table.startsWith(Namespace.ACCUMULO.name() + ".") &&
!table.equals("trace"))
- try {
- c.tableOperations().delete(table);
- } catch (TableNotFoundException e) {
- // don't care
- }
- }
- }
- }
-
- @After
public void tearDownShell() {
ts.shell.shutdown();
}
@@ -751,17 +553,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
while (!success) {
try {
ts.exec("insert a b c d -l foo", false, "does not have authorization",
true,
- new ErrorMessageCallback() {
- @Override
- public String getErrorMessage() {
- try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
- return "Current auths for root are: "
- + c.securityOperations().getUserAuthorizations("root");
- } catch (Exception e) {
- return "Could not check authorizations";
- }
- }
- });
+ new ErrorMessageCallback(getClientProps()));
success = true;
} catch (AssertionError e) {
Thread.sleep(500);
@@ -2306,615 +2098,4 @@ public class ShellServerIT extends
SharedMiniClusterBase {
// check that there are two files, with none having extra summary info
assertMatches(output,
"(?sm).*^.*total[:]2[,]\\s+missing[:]0[,]\\s+extra[:]0.*$.*");
}
-
- @Test
- public void testCreateTableWithLocalityGroups() throws Exception {
- final String table = getUniqueNames(1)[0];
- ts.exec("createtable " + table + " -l locg1=fam1,fam2", true);
- try (AccumuloClient accumuloClient =
Accumulo.newClient().from(getClientProps()).build()) {
- Map<String,Set<Text>> lMap =
accumuloClient.tableOperations().getLocalityGroups(table);
- Set<Text> expectedColFams = Set.of(new Text("fam1"), new Text("fam2"));
- for (Entry<String,Set<Text>> entry : lMap.entrySet()) {
- assertEquals("locg1", entry.getKey());
- assertTrue(entry.getValue().containsAll(expectedColFams));
- }
- ts.exec("deletetable -f " + table);
- }
- }
-
- /**
- * Due to the existing complexity of the createtable command, the
createtable help only displays
- * an example of setting one locality group. It is possible to set multiple
groups if needed. This
- * test verifies that capability.
- */
- @Test
- public void testCreateTableWithMultipleLocalityGroups() throws Exception {
- final String table = getUniqueNames(1)[0];
- ts.exec("createtable " + table + " -l locg1=fam1,fam2 locg2=colfam1",
true);
- try (AccumuloClient accumuloClient =
Accumulo.newClient().from(getClientProps()).build()) {
- Map<String,Set<Text>> lMap =
accumuloClient.tableOperations().getLocalityGroups(table);
- assertTrue(lMap.containsKey("locg1"));
- assertTrue(lMap.containsKey("locg2"));
- Set<Text> expectedColFams1 = Set.of(new Text("fam1"), new Text("fam2"));
- Set<Text> expectedColFams2 = Set.of(new Text("colfam1"));
- assertTrue(lMap.get("locg1").containsAll(expectedColFams1));
- assertTrue(lMap.get("locg2").containsAll(expectedColFams2));
- ts.exec("deletetable -f " + table);
- }
- }
-
- @Test
- public void testCreateTableWithLocalityGroupsBadArguments() throws
IOException {
- final String table = getUniqueNames(1)[0];
- ts.exec("createtable " + table + " -l locg1 fam1,fam2", false);
- ts.exec("createtable " + table + "-l", false);
- ts.exec("createtable " + table + " -l locg1 = fam1,fam2", false);
- ts.exec("createtable " + table + " -l locg1=fam1 ,fam2", false);
- ts.exec("createtable " + table + " -l locg1=fam1,fam2 locg1=fam3,fam4",
false);
- ts.exec("createtable " + table + " -l locg1=fam1,fam2 locg2=fam1", false);
- ts.exec("createtable " + table + " -l locg1", false);
- ts.exec("createtable " + table + " group=fam1", false);
- ts.exec("createtable " + table + "-l fam1,fam2", false);
- }
-
- @Test
- public void testCreateTableWithIterators() throws Exception {
- final String tmpTable = "tmpTable";
- final String table = getUniqueNames(1)[0];
-
- // create iterator profile
- // Will use tmpTable for creating profile since setshelliter is requiring
a table
- // even though its command line help indicates that it is optional. Likely
due to
- // the fact that setshelliter extends setiter, which does require a table
argument.
- ts.exec("createtable " + tmpTable, true);
- String output = ts.exec("tables");
- assertTrue(output.contains(tmpTable));
-
- ts.input.set("\n5000\n\n");
- ts.exec("setshelliter -n itname -p 10 -pn profile1 -ageoff", true);
- output = ts.exec("listshelliter");
- assertTrue(output.contains("Profile : profile1"));
-
- // create table making use of the iterator profile
- ts.exec("createtable " + table + " -i profile1:scan,minc", true);
- ts.exec("insert foo a b c", true);
- ts.exec("scan", true, "foo a:b []\tc");
- ts.exec("sleep 6", true);
- ts.exec("scan", true, "", true);
- ts.exec("deletetable -f " + table);
- ts.exec("deletetable -f " + tmpTable);
- }
-
- /**
- * Due to the existing complexity of the createtable command, the
createtable help only displays
- * an example of setting one iterator upon table creation. It is possible to
set multiple if
- * needed. This test verifies that capability.
- */
- @Test
- public void testCreateTableWithMultipleIterators() throws Exception {
- final String tmpTable = "tmpTable";
- final String table = getUniqueNames(1)[0];
-
- // create iterator profile
- // Will use tmpTable for creating profile since setshelliter is requiring
a table
- // even though its command line help indicates that it is optional. Likely
due to
- // the fact that setshelliter extends setiter, which does require a table
argument.
- ts.exec("createtable " + tmpTable, true);
- String output = ts.exec("tables");
- assertTrue(output.contains(tmpTable));
-
- ts.input.set("\n5000\n\n");
- ts.exec("setshelliter -n itname -p 10 -pn profile1 -ageoff", true);
- output = ts.exec("listshelliter");
- assertTrue(output.contains("Profile : profile1"));
-
- ts.input.set("2\n");
- ts.exec("setshelliter -n iter2 -p 11 -pn profile2 -vers", true);
- output = ts.exec("listshelliter");
- assertTrue(output.contains("Profile : profile2"));
-
- // create table making use of the iterator profiles
- ts.exec("createtable " + table + " -i profile1:scan,minc profile2:all ",
true);
- ts.exec("insert foo a b c", true);
- ts.exec("scan", true, "foo a:b []\tc");
- ts.exec("sleep 6", true);
- ts.exec("scan", true, "", true);
- output = ts.exec("listiter -t " + table + " -all");
- assertTrue(output.contains("Iterator itname, scan scope options"));
- assertTrue(output.contains("Iterator itname, minc scope options"));
- assertFalse(output.contains("Iterator itname, majc scope options"));
- assertTrue(output.contains("Iterator iter2, scan scope options"));
- assertTrue(output.contains("Iterator iter2, minc scope options"));
- assertTrue(output.contains("Iterator iter2, majc scope options"));
- ts.exec("deletetable -f " + table);
- ts.exec("deletetable -f " + tmpTable);
- }
-
- @Test
- public void testCreateTableWithIteratorsBadArguments() throws IOException {
- final String tmpTable = "tmpTable";
- final String table = getUniqueNames(1)[0];
- ts.exec("createtable " + tmpTable, true);
- String output = ts.exec("tables");
- assertTrue(output.contains(tmpTable));
- ts.input.set("\n5000\n\n");
- ts.exec("setshelliter -n itname -p 10 -pn profile1 -ageoff", true);
- output = ts.exec("listshelliter");
- assertTrue(output.contains("Profile : profile1"));
- // test various bad argument calls
- ts.exec("createtable " + table + " -i noprofile:scan,minc", false);
- ts.exec("createtable " + table + " -i profile1:scan,minc,all,majc", false);
- ts.exec("createtable " + table + " -i profile1:scan,all,majc", false);
- ts.exec("createtable " + table + " -i profile1:scan,min,majc", false);
- ts.exec("createtable " + table + " -i profile1:scan,max,all", false);
- ts.exec("createtable " + table + " -i profile1:", false);
- ts.exec("createtable " + table + " -i profile1: ", false);
- ts.exec("createtable " + table + " -i profile1:-scan", false);
- ts.exec("createtable " + table + " profile1:majc", false);
- ts.exec("createtable " + table + " -i profile1: all", false);
- ts.exec("createtable " + table + " -i profile1: All", false);
- ts.exec("createtable " + table + " -i profile1: scan", false);
- ts.exec("createtable " + table + " -i profile1:minc scan", false);
- ts.exec("createtable " + table + " -i profile1:minc,Scan", false);
- ts.exec("createtable " + table + " -i profile1:minc, scan", false);
- ts.exec("createtable " + table + " -i profile1:minc,,scan", false);
- ts.exec("createtable " + table + " -i profile1:minc,minc", false);
- ts.exec("createtable " + table + " -i profile1:minc,Minc", false);
- ts.exec("createtable " + table + " -i profile1:minc, ,scan", false);
- ts.exec("createtable " + table + "-i", false);
- ts.exec("createtable " + table + "-i ", false);
- ts.exec("deletetable -f " + tmpTable);
- }
-
- /**
- * Verify that table can be created in offline status and then be brought
online.
- */
- @Test
- public void testCreateTableOffline() throws IOException {
- final String tableName = getUniqueNames(1)[0];
- ts.exec("createtable " + tableName + " -o", true);
- String output = ts.exec("tables");
- assertTrue(output.contains(tableName));
- output = ts.exec("scan -t " + tableName, false, "is offline", true);
- assertTrue(output.contains("TableOfflineException"));
- ts.exec("table " + tableName, true);
- ts.exec("online", true);
- ts.exec("scan", true);
- ts.exec("deletetable -f " + tableName, true);
- }
-
- /**
- * Use shell to create a table with a supplied file containing splits.
- *
- * The splits will be contained in a file, sorted and un-encoded with no
repeats or blank lines.
- */
- @Test
- public void testCreateTableWithSplitsFile1()
- throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- generateSplitsFile(splitsFile, 1000, 12, false, false, true, false,
false);
- SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
- final String tableName = getUniqueNames(1)[0];
- ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
- Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
- assertEquals(expectedSplits, new TreeSet<>(createdSplits));
- } finally {
- Files.delete(Paths.get(splitsFile));
- }
- }
-
- /**
- * Use shell to create a table with a supplied file containing splits.
- *
- * The splits will be contained in a file, unsorted and un-encoded with no
repeats or blank lines.
- */
- @Test
- public void testCreateTableWithSplitsFile2()
- throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- generateSplitsFile(splitsFile, 300, 12, false, false, false, false,
false);
- SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
- final String tableName = getUniqueNames(1)[0];
- ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
- Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
- assertEquals(expectedSplits, new TreeSet<>(createdSplits));
- } finally {
- Files.delete(Paths.get(splitsFile));
- }
- }
-
- /**
- * Use shell to create a table with a supplied file containing splits.
- *
- * The splits will be contained in a file, sorted and encoded with no
repeats or blank lines.
- */
- @Test
- public void testCreateTableWithSplitsFile3()
- throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- generateSplitsFile(splitsFile, 100, 23, false, true, true, false, false);
- SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
- final String tableName = getUniqueNames(1)[0];
- ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
- Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
- assertEquals(expectedSplits, new TreeSet<>(createdSplits));
- } finally {
- Files.delete(Paths.get(splitsFile));
- }
- }
-
- /**
- * Use shell to create a table with a supplied file containing splits.
- *
- * The splits will be contained in a file, sorted and un-encoded with a
blank line and no repeats.
- */
- @Test
- public void testCreateTableWithSplitsFile4()
- throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- generateSplitsFile(splitsFile, 100, 31, false, false, true, true, false);
- SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
- final String tableName = getUniqueNames(1)[0];
- ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
- Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
- assertEquals(expectedSplits, new TreeSet<>(createdSplits));
- } finally {
- Files.delete(Paths.get(splitsFile));
- }
- }
-
- /**
- * Use shell to create a table with a supplied file containing splits.
- *
- * The splits will be contained in a file, sorted and un-encoded with a
blank line and no repeats.
- */
- @Test
- public void testCreateTableWithSplitsFile5()
- throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- generateSplitsFile(splitsFile, 100, 32, false, false, true, false, true);
- SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
- final String tableName = getUniqueNames(1)[0];
- ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
- Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
- assertEquals(expectedSplits, new TreeSet<>(createdSplits));
- } finally {
- Files.delete(Paths.get(splitsFile));
- }
- }
-
- /**
- * Use shell to create a table with a supplied file containing splits.
- *
- * The splits will be contained in a file, unsorted and un-encoded with a
blank line and repeats.
- */
- @Test
- public void testCreateTableWithSplitsFile6()
- throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- generateSplitsFile(splitsFile, 100, 12, false, false, false, true, true);
- SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
- final String tableName = getUniqueNames(1)[0];
- ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
- Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
- assertEquals(expectedSplits, new TreeSet<>(createdSplits));
- } finally {
- Files.delete(Paths.get(splitsFile));
- }
- }
-
- /**
- * Use shell to create a table with a supplied file containing splits.
- *
- * The splits will be contained in a file, sorted and encoded with a blank
line and repeats.
- */
- @Test
- public void testCreateTableWithSplitsFile7()
- throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- generateSplitsFile(splitsFile, 100, 12, false, false, true, true, true);
- SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
- final String tableName = getUniqueNames(1)[0];
- ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
- Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
- assertEquals(expectedSplits, new TreeSet<>(createdSplits));
- } finally {
- Files.delete(Paths.get(splitsFile));
- }
- }
-
- /**
- * Use shell to create a table with a supplied file containing splits.
- *
- * The splits file will be empty.
- */
- @Test
- public void testCreateTableWithEmptySplitFile()
- throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- generateSplitsFile(splitsFile, 0, 0, false, false, false, false, false);
- final String tableName = getUniqueNames(1)[0];
- ts.exec("createtable " + tableName + " -sf " + splitsFile, false);
- assertThrows(TableNotFoundException.class,
- () -> client.tableOperations().listSplits(tableName));
- } finally {
- Files.delete(Paths.get(splitsFile));
- }
- }
-
- /**
- * Use shell to create a table that used splits from another table.
- */
- @Test
- public void testCreateTableWithCopySplitsFromOtherTable()
- throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- // create a table and add some splits
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- final String[] tableNames = getUniqueNames(2);
- final String tableName0 = tableNames[0];
- final String tableName2 = tableNames[1];
-
- ts.exec("createtable " + tableName0, true);
- String output = ts.exec("tables", true);
- assertTrue(output.contains(tableName0));
- ts.exec("table " + tableName0, true);
- // add splits to this table using the addsplits command.
- List<Text> splits = new ArrayList<>();
- splits.add(new Text("ccccc"));
- splits.add(new Text("fffff"));
- splits.add(new Text("mmmmm"));
- splits.add(new Text("sssss"));
- ts.exec("addsplits " + splits.get(0) + " " + splits.get(1) + " " +
splits.get(2) + " "
- + splits.get(3), true);
- // Now create a table that will used the previous tables splits and
create them at table
- // creation
- ts.exec("createtable " + tableName2 + " --copy-splits " + tableName0,
true);
- ts.exec("table " + tableName0, true);
- String tablesOutput = ts.exec("tables", true);
- assertTrue(tablesOutput.contains(tableName2));
- Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName2);
- assertEquals(new TreeSet<>(splits), new TreeSet<>(createdSplits));
- ts.exec("deletetable -f " + tableName0, true);
- ts.exec("deletetable -f " + tableName2, true);
- }
- }
-
- /**
- * Use shell to create a table with a supplied file containing splits.
- *
- * The splits will be contained in a file, sorted and encoded with no
repeats or blank lines.
- */
- @Test
- public void testCreateTableWithBinarySplitsFile1()
- throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- generateSplitsFile(splitsFile, 200, 12, true, true, true, false, false);
- SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
- final String tableName = getUniqueNames(1)[0];
- ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
- Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
- assertEquals(expectedSplits, new TreeSet<>(createdSplits));
- } finally {
- Files.delete(Paths.get(splitsFile));
- }
- }
-
- /**
- * Use shell to create a table with a supplied file containing splits.
- *
- * The splits will be contained in a file, unsorted and encoded with no
repeats or blank lines.
- */
- @Test
- public void testCreateTableWithBinarySplitsFile2()
- throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- generateSplitsFile(splitsFile, 300, 12, true, true, false, false, false);
- SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
- final String tableName = getUniqueNames(1)[0];
- ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
- Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
- assertEquals(expectedSplits, new TreeSet<>(createdSplits));
- } finally {
- Files.delete(Paths.get(splitsFile));
- }
- }
-
- /**
- * Use shell to create a table with a supplied file containing splits.
- *
- * The splits will be contained in a file, sorted and encoded with no
repeats or blank lines.
- */
- @Test
- public void testCreateTableWithBinarySplitsFile3()
- throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- generateSplitsFile(splitsFile, 100, 23, true, true, true, false, false);
- SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
- final String tableName = getUniqueNames(1)[0];
- ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
- Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
- assertEquals(expectedSplits, new TreeSet<>(createdSplits));
- } finally {
- Files.delete(Paths.get(splitsFile));
- }
- }
-
- /**
- * Use shell to create a table with a supplied file containing splits.
- *
- * The splits will be contained in a file, sorted and encoded with a blank
line and no repeats.
- */
- @Test
- public void testCreateTableWithBinarySplitsFile4()
- throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- generateSplitsFile(splitsFile, 100, 31, true, true, true, true, false);
- SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
- final String tableName = getUniqueNames(1)[0];
- ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
- Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
- assertEquals(expectedSplits, new TreeSet<>(createdSplits));
- } finally {
- Files.delete(Paths.get(splitsFile));
- }
- }
-
- /**
- * Use shell to create a table with a supplied file containing splits.
- *
- * The splits will be contained in a file, sorted and encoded with a blank
line and no repeats.
- */
- @Test
- public void testCreateTableWithBinarySplitsFile5()
- throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- generateSplitsFile(splitsFile, 100, 32, true, true, true, false, true);
- SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
- final String tableName = getUniqueNames(1)[0];
- ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
- Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
- assertEquals(expectedSplits, new TreeSet<>(createdSplits));
- } finally {
- Files.delete(Paths.get(splitsFile));
- }
- }
-
- /**
- * Use shell to create a table with a supplied file containing splits.
- *
- * The splits will be contained in a file, unsorted and encoded with a blank
line and repeats.
- */
- @Test
- public void testCreateTableWithBinarySplitsFile6()
- throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- generateSplitsFile(splitsFile, 100, 12, true, true, false, true, true);
- SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
- final String tableName = getUniqueNames(1)[0];
- ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
- Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
- assertEquals(expectedSplits, new TreeSet<>(createdSplits));
- } finally {
- Files.delete(Paths.get(splitsFile));
- }
- }
-
- /**
- * Use shell to create a table with a supplied file containing splits.
- *
- * The splits will be contained in a file, sorted and encoded with a blank
line and repeats.
- */
- @Test
- public void testCreateTableWithBinarySplitsFile7()
- throws IOException, AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- String splitsFile = System.getProperty("user.dir") + "/target/splitFile";
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- generateSplitsFile(splitsFile, 100, 12, true, true, true, true, true);
- SortedSet<Text> expectedSplits = readSplitsFromFile(splitsFile);
- final String tableName = getUniqueNames(1)[0];
- ts.exec("createtable " + tableName + " -sf " + splitsFile, true);
- Collection<Text> createdSplits =
client.tableOperations().listSplits(tableName);
- assertEquals(expectedSplits, new TreeSet<>(createdSplits));
- } finally {
- Files.delete(Paths.get(splitsFile));
- }
- }
-
- private SortedSet<Text> readSplitsFromFile(final String splitsFile) throws
IOException {
- SortedSet<Text> splits = new TreeSet<>();
- try (BufferedReader reader = newBufferedReader(Paths.get(splitsFile))) {
- String split;
- while ((split = reader.readLine()) != null) {
- Text unencodedString = decode(split);
- if (unencodedString != null)
- splits.add(unencodedString);
- }
- }
- return splits;
- }
-
- private void generateSplitsFile(final String splitsFile, final int numItems,
final int len,
- final boolean binarySplits, final boolean encoded, final boolean sort,
- final boolean addBlankLine, final boolean repeat) throws IOException {
-
- java.nio.file.Path splitsPath = java.nio.file.Paths.get(splitsFile);
- int insertAt = (len % 2 == 0) ? len / 2 : (len + 1) / 2;
- Collection<Text> sortedSplits = null;
- Collection<Text> randomSplits;
-
- if (binarySplits)
- randomSplits = generateBinarySplits(numItems, len);
- else
- randomSplits = generateNonBinarySplits(numItems, len);
-
- if (sort)
- sortedSplits = new TreeSet<>(randomSplits);
-
- try (BufferedWriter writer = Files.newBufferedWriter(splitsPath, UTF_8)) {
- int cnt = 0;
- Collection<Text> splits;
- if (sort)
- splits = sortedSplits;
- else
- splits = randomSplits;
-
- for (Text text : splits) {
- if (addBlankLine && cnt++ == insertAt)
- writer.write('\n');
- writer.write(encode(text, encoded) + '\n');
- if (repeat)
- writer.write(encode(text, encoded) + '\n');
- }
- }
- }
-
- private Collection<Text> generateNonBinarySplits(final int numItems, final
int len) {
- Set<Text> splits = new HashSet<>();
- for (int i = 0; i < numItems; i++) {
- splits.add(getRandomText(len));
- }
- return splits;
- }
-
- private Collection<Text> generateBinarySplits(final int numItems, final int
len) {
- Set<Text> splits = new HashSet<>();
- for (int i = 0; i < numItems; i++) {
- byte[] split = new byte[len];
- random.nextBytes(split);
- splits.add(new Text(split));
- }
- return splits;
- }
-
- private Text getRandomText(final int len) {
- int desiredLen = Math.min(len, 32);
- return new Text(
- String.valueOf(UUID.randomUUID()).replaceAll("-", "").substring(0,
desiredLen - 1));
- }
-
- private static String encode(final Text text, final boolean encode) {
- if (text.toString().isBlank())
- return null;
- return encode ?
Base64.getEncoder().encodeToString(TextUtil.getBytes(text)) : text.toString();
- }
-
- private Text decode(final String text) {
- if (requireNonNull(text).isBlank())
- return null;
- return new Text(text);
- }
}
