This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new 7db5a0d Clean up Volume chooser ITs
7db5a0d is described below
commit 7db5a0d8b4ca9fb3abde81c4f1caa6ff24af5685
Author: Mike Miller <mmil...@apache.org>
AuthorDate: Tue Jan 22 18:14:06 2019 -0500
Clean up Volume chooser ITs
---
.../org/apache/accumulo/test/VolumeChooserIT.java | 107 +++++-------------
.../java/org/apache/accumulo/test/VolumeIT.java | 124 ++++++++-------------
2 files changed, 76 insertions(+), 155 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
b/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
index 620ce4c..cbbcd84 100644
--- a/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
@@ -64,7 +64,8 @@ public class VolumeChooserIT extends ConfigurableMacBase {
private File volDirBase;
@SuppressWarnings("unused")
private Path v1, v2, v3, v4;
- private static String[] rows =
"a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
+ public static String[] alpha_rows =
"a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z"
+ .split(",");
private String namespace1;
private String namespace2;
private String systemPreferredVolumes;
@@ -127,28 +128,28 @@ public class VolumeChooserIT extends ConfigurableMacBase {
throws TableNotFoundException, AccumuloException,
AccumuloSecurityException {
// Add 10 splits to the table
SortedSet<Text> partitions = new TreeSet<>();
- for (String s : rows)
+ for (String s : alpha_rows)
partitions.add(new Text(s));
accumuloClient.tableOperations().addSplits(tableName, partitions);
}
public static void writeAndReadData(AccumuloClient accumuloClient, String
tableName)
throws Exception {
- writeDataToTable(accumuloClient, tableName);
+ writeDataToTable(accumuloClient, tableName, alpha_rows);
// Write the data to disk, read it back
accumuloClient.tableOperations().flush(tableName, null, null, true);
try (Scanner scanner = accumuloClient.createScanner(tableName,
Authorizations.EMPTY)) {
int i = 0;
for (Entry<Key,Value> entry : scanner) {
- assertEquals("Data read is not data written", rows[i++],
+ assertEquals("Data read is not data written", alpha_rows[i++],
entry.getKey().getRow().toString());
}
}
}
- public static void writeDataToTable(AccumuloClient accumuloClient, String
tableName)
- throws Exception {
+ public static void writeDataToTable(AccumuloClient accumuloClient, String
tableName,
+ String[] rows) throws Exception {
// Write some data to the table
BatchWriter bw = accumuloClient.createBatchWriter(tableName, new
BatchWriterConfig());
for (String s : rows) {
@@ -259,28 +260,18 @@ public class VolumeChooserIT extends ConfigurableMacBase {
log.info("Starting twoTablesPreferredVolumeChooser");
// Create namespace
- try (AccumuloClient accumuloClient = createClient()) {
- accumuloClient.namespaceOperations().create(namespace1);
-
+ try (AccumuloClient c = createClient()) {
// Set properties on the namespace
// namespace 1 -> v2
- accumuloClient.namespaceOperations().setProperty(namespace1,
- PerTableVolumeChooser.TABLE_VOLUME_CHOOSER,
PreferredVolumeChooser.class.getName());
- accumuloClient.namespaceOperations().setProperty(namespace1,
- PreferredVolumeChooser.TABLE_PREFERRED_VOLUMES, v2.toString());
+ configureNamespace(c, PreferredVolumeChooser.class.getName(),
v2.toString(), namespace1);
// Create table1 on namespace1
- verifyVolumesForWritesToNewTable(accumuloClient, namespace1,
v2.toString());
+ verifyVolumesForWritesToNewTable(c, namespace1, v2.toString());
- accumuloClient.namespaceOperations().create(namespace2);
- // Set properties on the namespace
- accumuloClient.namespaceOperations().setProperty(namespace2,
- PerTableVolumeChooser.TABLE_VOLUME_CHOOSER,
PreferredVolumeChooser.class.getName());
- accumuloClient.namespaceOperations().setProperty(namespace2,
- PreferredVolumeChooser.TABLE_PREFERRED_VOLUMES, v1.toString());
+ configureNamespace(c, PreferredVolumeChooser.class.getName(),
v1.toString(), namespace2);
// Create table2 on namespace2
- verifyVolumesForWritesToNewTable(accumuloClient, namespace2,
v1.toString());
+ verifyVolumesForWritesToNewTable(c, namespace2, v1.toString());
}
}
@@ -292,72 +283,34 @@ public class VolumeChooserIT extends ConfigurableMacBase {
// Create namespace
try (AccumuloClient accumuloClient = createClient()) {
- accumuloClient.namespaceOperations().create(namespace1);
-
- // Set properties on the namespace
- accumuloClient.namespaceOperations().setProperty(namespace1,
- PerTableVolumeChooser.TABLE_VOLUME_CHOOSER,
RandomVolumeChooser.class.getName());
-
- // Create table1 on namespace1
- String tableName = namespace1 + ".1";
- accumuloClient.tableOperations().create(tableName);
- Table.ID tableID =
Table.ID.of(accumuloClient.tableOperations().tableIdMap().get(tableName));
-
- // Add 10 splits to the table
- addSplits(accumuloClient, tableName);
- // Write some data to the table
- writeAndReadData(accumuloClient, tableName);
- // Verify the new files are written to the Volumes specified
-
- verifyVolumes(accumuloClient, TabletsSection.getRange(tableID), v1 + ","
+ v2 + "," + v4);
+ createAndVerify(accumuloClient, namespace1, v1 + "," + v2 + "," + v4);
+ createAndVerify(accumuloClient, namespace2, v1 + "," + v2 + "," + v4);
+ }
+ }
- accumuloClient.namespaceOperations().create(namespace2);
+ private void createAndVerify(AccumuloClient accumuloClient, String ns,
String expectedVolumes)
+ throws Exception {
+ accumuloClient.namespaceOperations().create(ns);
- // Set properties on the namespace
- accumuloClient.namespaceOperations().setProperty(namespace2,
- PerTableVolumeChooser.TABLE_VOLUME_CHOOSER,
RandomVolumeChooser.class.getName());
+ // Set properties on the namespace
+ accumuloClient.namespaceOperations().setProperty(ns,
PerTableVolumeChooser.TABLE_VOLUME_CHOOSER,
+ RandomVolumeChooser.class.getName());
- // Create table2 on namespace2
- String tableName2 = namespace2 + ".1";
- accumuloClient.tableOperations().create(tableName2);
- Table.ID tableID2 = Table.ID
- .of(accumuloClient.tableOperations().tableIdMap().get(tableName2));
-
- // / Add 10 splits to the table
- addSplits(accumuloClient, tableName2);
- // Write some data to the table
- writeAndReadData(accumuloClient, tableName2);
- // Verify the new files are written to the Volumes specified
- verifyVolumes(accumuloClient, TabletsSection.getRange(tableID2), v1 +
"," + v2 + "," + v4);
- }
+ verifyVolumesForWritesToNewTable(accumuloClient, ns, expectedVolumes);
}
- // Test that uses two tables with 10 split points each. The first uses the
RandomVolumeChooser and
- // the second uses the
- // StaticVolumeChooser to choose volumes.
+ // Test that uses 2 tables with 10 split points each. The first uses the
RandomVolumeChooser and
+ // the second uses the StaticVolumeChooser to choose volumes.
@Test
public void twoTablesDiffChoosers() throws Exception {
log.info("Starting twoTablesDiffChoosers");
// Create namespace
- try (AccumuloClient accumuloClient = createClient()) {
- accumuloClient.namespaceOperations().create(namespace1);
-
- // Set properties on the namespace
- accumuloClient.namespaceOperations().setProperty(namespace1,
- PerTableVolumeChooser.TABLE_VOLUME_CHOOSER,
RandomVolumeChooser.class.getName());
-
- // Create table1 on namespace1
- verifyVolumesForWritesToNewTable(accumuloClient, namespace1, v1 + "," +
v2 + "," + v4);
- accumuloClient.namespaceOperations().create(namespace2);
-
- accumuloClient.namespaceOperations().setProperty(namespace2,
- PerTableVolumeChooser.TABLE_VOLUME_CHOOSER,
PreferredVolumeChooser.class.getName());
- accumuloClient.namespaceOperations().setProperty(namespace2,
- PreferredVolumeChooser.TABLE_PREFERRED_VOLUMES, v1.toString());
-
+ try (AccumuloClient c = createClient()) {
+ createAndVerify(c, namespace1, v1 + "," + v2 + "," + v4);
+ configureNamespace(c, PreferredVolumeChooser.class.getName(),
v1.toString(), namespace2);
// Create table2 on namespace2
- verifyVolumesForWritesToNewTable(accumuloClient, namespace2,
v1.toString());
+ verifyVolumesForWritesToNewTable(c, namespace2, v1.toString());
}
}
@@ -383,7 +336,7 @@ public class VolumeChooserIT extends ConfigurableMacBase {
accumuloClient.tableOperations().create(tableName);
VolumeChooserIT.addSplits(accumuloClient, tableName);
- VolumeChooserIT.writeDataToTable(accumuloClient, tableName);
+ VolumeChooserIT.writeDataToTable(accumuloClient, tableName, alpha_rows);
// should only go to v2 as per configuration in configure()
VolumeChooserIT.verifyWaLogVolumes(accumuloClient, new Range(),
v2.toString());
}
diff --git a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
index c9626f6..4347d82 100644
--- a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
@@ -39,7 +39,6 @@ import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
@@ -85,10 +84,9 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
public class VolumeIT extends ConfigurableMacBase {
- private static final Text EMPTY = new Text();
- private static final Value EMPTY_VALUE = new Value(new byte[] {});
private File volDirBase;
- private Path v1, v2;
+ private Path v1, v2, v3;
+ private List<String> expected = new ArrayList<>();
@Override
protected int defaultTimeoutSeconds() {
@@ -104,6 +102,13 @@ public class VolumeIT extends ConfigurableMacBase {
File v2f = new File(volDirBase, "v2");
v1 = new Path("file://" + v1f.getAbsolutePath());
v2 = new Path("file://" + v2f.getAbsolutePath());
+ File v3f = new File(volDirBase, "v3");
+ v3 = new Path("file://" + v3f.getAbsolutePath());
+ // setup expected rows
+ for (int i = 0; i < 100; i++) {
+ String row = String.format("%06d", i * 100 + 3);
+ expected.add(row + ":cf1:cq1:1");
+ }
// Run MAC on two locations in the local file system
URI v1Uri = v1.toUri();
@@ -131,20 +136,13 @@ public class VolumeIT extends ConfigurableMacBase {
partitions.add(new Text(s));
accumuloClient.tableOperations().addSplits(tableName, partitions);
// scribble over the splits
- BatchWriter bw = accumuloClient.createBatchWriter(tableName, new
BatchWriterConfig());
- String[] rows =
"a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
- for (String s : rows) {
- Mutation m = new Mutation(new Text(s));
- m.put(EMPTY, EMPTY, EMPTY_VALUE);
- bw.addMutation(m);
- }
- bw.close();
+ VolumeChooserIT.writeDataToTable(accumuloClient, tableName,
VolumeChooserIT.alpha_rows);
// write the data to disk, read it back
accumuloClient.tableOperations().flush(tableName, null, null, true);
try (Scanner scanner = accumuloClient.createScanner(tableName,
Authorizations.EMPTY)) {
int i = 0;
for (Entry<Key,Value> entry : scanner) {
- assertEquals(rows[i++], entry.getKey().getRow().toString());
+ assertEquals(VolumeChooserIT.alpha_rows[i++],
entry.getKey().getRow().toString());
}
}
// verify the new files are written to the different volumes
@@ -210,9 +208,7 @@ public class VolumeIT extends ConfigurableMacBase {
BatchWriter bw = accumuloClient.createBatchWriter(tableName, new
BatchWriterConfig());
// create two files in each tablet
-
- String[] rows =
"a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
- for (String s : rows) {
+ for (String s : VolumeChooserIT.alpha_rows) {
Mutation m = new Mutation(s);
m.put("cf1", "cq1", "1");
bw.addMutation(m);
@@ -222,7 +218,7 @@ public class VolumeIT extends ConfigurableMacBase {
bw.flush();
accumuloClient.tableOperations().flush(tableName, null, null, true);
- for (String s : rows) {
+ for (String s : VolumeChooserIT.alpha_rows) {
Mutation m = new Mutation(s);
m.put("cf1", "cq1", "2");
bw.addMutation(m);
@@ -284,21 +280,9 @@ public class VolumeIT extends ConfigurableMacBase {
public void testAddVolumes() throws Exception {
try (AccumuloClient client = createClient()) {
String[] tableNames = getUniqueNames(2);
-
- // grab this before shutting down cluster
- String uuid = client.getInstanceID();
-
- verifyVolumesUsed(client, tableNames[0], false, v1, v2);
-
- assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
- cluster.stop();
-
PropertiesConfiguration conf = new PropertiesConfiguration();
- conf.load(cluster.getAccumuloPropertiesPath());
- File v3f = new File(volDirBase, "v3");
- assertTrue(v3f.mkdir() || v3f.isDirectory());
- Path v3 = new Path("file://" + v3f.getAbsolutePath());
+ String uuid = verifyAndShutdownCluster(client, conf, tableNames[0]);
conf.setProperty(Property.INSTANCE_VOLUMES.getKey(), v1 + "," + v2 + ","
+ v3);
conf.save(cluster.getAccumuloPropertiesPath());
@@ -306,14 +290,7 @@ public class VolumeIT extends ConfigurableMacBase {
// initialize volume
assertEquals(0, cluster.exec(Initialize.class,
"--add-volumes").waitFor());
- // check that all volumes are initialized
- for (Path volumePath : Arrays.asList(v1, v2, v3)) {
- FileSystem fs =
volumePath.getFileSystem(CachedConfiguration.getInstance());
- Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
- FileStatus[] iids = fs.listStatus(vp);
- assertEquals(1, iids.length);
- assertEquals(uuid, iids[0].getPath().getName());
- }
+ checkVolumesInitialized(Arrays.asList(v1, v2, v3), uuid);
// start cluster and verify that new volume is used
cluster.start();
@@ -322,26 +299,28 @@ public class VolumeIT extends ConfigurableMacBase {
}
}
- @Test
- public void testNonConfiguredVolumes() throws Exception {
+ // grab uuid before shutting down cluster
+ private String verifyAndShutdownCluster(AccumuloClient c,
PropertiesConfiguration conf,
+ String tableName) throws Exception {
+ String uuid = c.getInstanceID();
- String[] tableNames = getUniqueNames(2);
+ verifyVolumesUsed(c, tableName, false, v1, v2);
- // grab this before shutting down cluster
- try (AccumuloClient client = createClient()) {
- String uuid = client.getInstanceID();
+ assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ cluster.stop();
- verifyVolumesUsed(client, tableNames[0], false, v1, v2);
+ conf.load(cluster.getAccumuloPropertiesPath());
+ return uuid;
+ }
- assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
- cluster.stop();
+ @Test
+ public void testNonConfiguredVolumes() throws Exception {
- PropertiesConfiguration conf = new PropertiesConfiguration();
- conf.load(cluster.getAccumuloPropertiesPath());
+ String[] tableNames = getUniqueNames(2);
+ PropertiesConfiguration conf = new PropertiesConfiguration();
- File v3f = new File(volDirBase, "v3");
- assertTrue(v3f.mkdir() || v3f.isDirectory());
- Path v3 = new Path("file://" + v3f.getAbsolutePath());
+ try (AccumuloClient client = createClient()) {
+ String uuid = verifyAndShutdownCluster(client, conf, tableNames[0]);
conf.setProperty(Property.INSTANCE_VOLUMES.getKey(), v2 + "," + v3);
conf.save(cluster.getAccumuloPropertiesPath());
@@ -349,27 +328,12 @@ public class VolumeIT extends ConfigurableMacBase {
// initialize volume
assertEquals(0, cluster.exec(Initialize.class,
"--add-volumes").waitFor());
- // check that all volumes are initialized
- for (Path volumePath : Arrays.asList(v1, v2, v3)) {
- FileSystem fs =
volumePath.getFileSystem(CachedConfiguration.getInstance());
- Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
- FileStatus[] iids = fs.listStatus(vp);
- assertEquals(1, iids.length);
- assertEquals(uuid, iids[0].getPath().getName());
- }
+ checkVolumesInitialized(Arrays.asList(v1, v2, v3), uuid);
// start cluster and verify that new volume is used
cluster.start();
- // Make sure we can still read the tables (tableNames[0] is very likely
to have a file still
- // on
- // v1)
- List<String> expected = new ArrayList<>();
- for (int i = 0; i < 100; i++) {
- String row = String.format("%06d", i * 100 + 3);
- expected.add(row + ":cf1:cq1:1");
- }
-
+ // verify we can still read the tables (tableNames[0] is likely to have
a file still on v1)
verifyData(expected, client.createScanner(tableNames[0],
Authorizations.EMPTY));
// v1 should not have any data for tableNames[1]
@@ -377,9 +341,19 @@ public class VolumeIT extends ConfigurableMacBase {
}
}
- private void writeData(String tableName, AccumuloClient client)
- throws AccumuloException, AccumuloSecurityException,
TableExistsException,
- TableNotFoundException, MutationsRejectedException {
+ // check that all volumes are initialized
+ private void checkVolumesInitialized(List<Path> volumes, String uuid) throws
Exception {
+ for (Path volumePath : volumes) {
+ FileSystem fs =
volumePath.getFileSystem(CachedConfiguration.getInstance());
+ Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
+ FileStatus[] iids = fs.listStatus(vp);
+ assertEquals(1, iids.length);
+ assertEquals(uuid, iids[0].getPath().getName());
+ }
+ }
+
+ private void writeData(String tableName, AccumuloClient client) throws
AccumuloException,
+ AccumuloSecurityException, TableExistsException, TableNotFoundException {
TreeSet<Text> splits = new TreeSet<>();
for (int i = 1; i < 100; i++) {
splits.add(new Text(String.format("%06d", i * 100)));
@@ -402,12 +376,6 @@ public class VolumeIT extends ConfigurableMacBase {
private void verifyVolumesUsed(AccumuloClient client, String tableName,
boolean shouldExist,
Path... paths) throws Exception {
- List<String> expected = new ArrayList<>();
- for (int i = 0; i < 100; i++) {
- String row = String.format("%06d", i * 100 + 3);
- expected.add(row + ":cf1:cq1:1");
- }
-
if (!client.tableOperations().exists(tableName)) {
assertFalse(shouldExist);
