Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Conflicts:
examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
test/system/auto/simple/examples.py
In addition to merging from 1.4 also fixed a lot of issues with examples
functional test that were not issues in 1.4.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c175bd31
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c175bd31
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c175bd31
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: c175bd31cfb18a081e328ccf5b172f67fcea7491
Parents: 17ac692 1fe5f7f
Author: Keith Turner <[email protected]>
Authored: Mon Nov 25 13:14:32 2013 -0500
Committer: Keith Turner <[email protected]>
Committed: Mon Nov 25 13:14:32 2013 -0500
----------------------------------------------------------------------
.../simple/client/RandomBatchScanner.java | 21 ++--
.../examples/simple/mapreduce/TableToFile.java | 2 +-
test/system/auto/simple/examples.py | 103 ++++++++++---------
3 files changed, 73 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c175bd31/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
----------------------------------------------------------------------
diff --cc
examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
index 5194b67,0000000..2ae82b4
mode 100644,000000..100644
---
a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
+++
b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
@@@ -1,225 -1,0 +1,234 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.client;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+import java.util.Random;
+
+import org.apache.accumulo.core.cli.BatchScannerOpts;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Internal class used to verify validity of data read.
+ */
+class CountingVerifyingReceiver {
+ private static final Logger log =
Logger.getLogger(CountingVerifyingReceiver.class);
+
+ long count = 0;
+ int expectedValueSize = 0;
+ HashMap<Text,Boolean> expectedRows;
+
+ CountingVerifyingReceiver(HashMap<Text,Boolean> expectedRows, int
expectedValueSize) {
+ this.expectedRows = expectedRows;
+ this.expectedValueSize = expectedValueSize;
+ }
+
+ public void receive(Key key, Value value) {
+
+ String row = key.getRow().toString();
+ long rowid = Integer.parseInt(row.split("_")[1]);
+
+ byte expectedValue[] = RandomBatchWriter.createValue(rowid,
expectedValueSize);
+
+ if (!Arrays.equals(expectedValue, value.get())) {
+ log.error("Got unexpected value for " + key + " expected : " + new
String(expectedValue) + " got : " + new String(value.get()));
+ }
+
+ if (!expectedRows.containsKey(key.getRow())) {
+ log.error("Got unexpected key " + key);
+ } else {
+ expectedRows.put(key.getRow(), true);
+ }
+
+ count++;
+ }
+}
+
+/**
+ * Simple example for reading random batches of data from Accumulo. See
docs/examples/README.batch for instructions.
+ */
+public class RandomBatchScanner {
+ private static final Logger log =
Logger.getLogger(CountingVerifyingReceiver.class);
+
+ /**
+ * Generate a number of ranges, each covering a single random row.
+ *
+ * @param num
+ * the number of ranges to generate
+ * @param min
+ * the minimum row that will be generated
+ * @param max
+ * the maximum row that will be generated
+ * @param r
+ * a random number generator
+ * @param ranges
+ * a set in which to store the generated ranges
+ * @param expectedRows
+ * a map in which to store the rows covered by the ranges
(initially mapped to false)
+ */
+ static void generateRandomQueries(int num, long min, long max, Random r,
HashSet<Range> ranges, HashMap<Text,Boolean> expectedRows) {
+ log.info(String.format("Generating %,d random queries...", num));
+ while (ranges.size() < num) {
+ long rowid = (Math.abs(r.nextLong()) % (max - min)) + min;
+
+ Text row1 = new Text(String.format("row_%010d", rowid));
+
+ Range range = new Range(new Text(row1));
+ ranges.add(range);
+ expectedRows.put(row1, false);
+ }
+
+ log.info("finished");
+ }
+
+ /**
+ * Prints a count of the number of rows mapped to false.
+ *
+ * @param expectedRows
++ * @return boolean indicating "were all the rows found?"
+ */
- private static void printRowsNotFound(HashMap<Text,Boolean> expectedRows) {
++ private static boolean checkAllRowsFound(HashMap<Text,Boolean>
expectedRows) {
+ int count = 0;
++ boolean allFound = true;
+ for (Entry<Text,Boolean> entry : expectedRows.entrySet())
+ if (!entry.getValue())
+ count++;
+
- if (count > 0)
++ if (count > 0) {
+ log.warn("Did not find " + count + " rows");
++ allFound = false;
++ }
++ return allFound;
+ }
+
+ /**
+ * Generates a number of random queries, verifies that the key/value pairs
returned were in the queried ranges and that the values were generated by
+ * {@link RandomBatchWriter#createValue(long, int)}. Prints information
about the results.
+ *
+ * @param num
+ * the number of queries to generate
+ * @param min
+ * the min row to query
+ * @param max
+ * the max row to query
+ * @param evs
+ * the expected size of the values
+ * @param r
+ * a random number generator
+ * @param tsbr
+ * a batch scanner
++ * @return boolean indicating "did the queries go fine?"
+ */
- static void doRandomQueries(int num, long min, long max, int evs, Random r,
BatchScanner tsbr) {
++ static boolean doRandomQueries(int num, long min, long max, int evs, Random
r, BatchScanner tsbr) {
+
+ HashSet<Range> ranges = new HashSet<Range>(num);
+ HashMap<Text,Boolean> expectedRows = new
java.util.HashMap<Text,Boolean>();
+
+ generateRandomQueries(num, min, max, r, ranges, expectedRows);
+
+ tsbr.setRanges(ranges);
+
+ CountingVerifyingReceiver receiver = new
CountingVerifyingReceiver(expectedRows, evs);
+
+ long t1 = System.currentTimeMillis();
+
+ for (Entry<Key,Value> entry : tsbr) {
+ receiver.receive(entry.getKey(), entry.getValue());
+ }
+
+ long t2 = System.currentTimeMillis();
+
+ log.info(String.format("%6.2f lookups/sec %6.2f secs%n", num / ((t2 - t1)
/ 1000.0), ((t2 - t1) / 1000.0)));
+ log.info(String.format("num results : %,d%n", receiver.count));
+
- printRowsNotFound(expectedRows);
++ return checkAllRowsFound(expectedRows);
+ }
+
+ public static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--min", description="miniumum row that will be
generated")
+ long min = 0;
+ @Parameter(names="--max", description="maximum ow that will be generated")
+ long max = 0;
+ @Parameter(names="--num", required=true, description="number of ranges to
generate")
+ int num = 0;
+ @Parameter(names="--size", required=true, description="size of the value
to write")
+ int size = 0;
+ @Parameter(names="--seed", description="seed for pseudo-random number
generator")
+ Long seed = null;
+ }
+
+ /**
+ * Scans over a specified number of entries to Accumulo using a {@link
BatchScanner}. Completes scans twice to compare times for a fresh query with
those for
+ * a repeated query which has cached metadata and connections already
established.
+ *
+ * @param args
+ * @throws AccumuloException
+ * @throws AccumuloSecurityException
+ * @throws TableNotFoundException
+ */
+ public static void main(String[] args) throws AccumuloException,
AccumuloSecurityException, TableNotFoundException {
+ Opts opts = new Opts();
+ BatchScannerOpts bsOpts = new BatchScannerOpts();
+ opts.parseArgs(RandomBatchScanner.class.getName(), args, bsOpts);
+
+ Connector connector = opts.getConnector();
+ BatchScanner batchReader = connector.createBatchScanner(opts.tableName,
opts.auths, bsOpts.scanThreads);
+ batchReader.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
+
+ Random r;
+ if (opts.seed == null)
+ r = new Random();
+ else
+ r = new Random(opts.seed);
+
+ // do one cold
- doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
++ boolean status = doRandomQueries(opts.num, opts.min, opts.max, opts.size,
r, batchReader);
+
+ System.gc();
+ System.gc();
+ System.gc();
+
+ if (opts.seed == null)
+ r = new Random();
+ else
+ r = new Random(opts.seed);
+
+ // do one hot (connections already established, metadata table cached)
- doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
++ status = status && doRandomQueries(opts.num, opts.min, opts.max,
opts.size, r, batchReader);
+
+ batchReader.close();
++ if (!status) {
++ System.exit(1);
++ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c175bd31/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
----------------------------------------------------------------------
diff --cc
examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
index 4951852,0000000..d8eedef
mode 100644,000000..100644
---
a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
+++
b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
@@@ -1,130 -1,0 +1,130 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.mapreduce;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Map;
+
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.format.DefaultFormatter;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Takes a table and outputs the specified column to a set of part files on
hdfs accumulo accumulo.examples.mapreduce.TableToFile <username> <password>
+ * <tablename> <column> <hdfs-output-path>
+ */
+public class TableToFile extends Configured implements Tool {
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names = "--output", description = "output directory", required
= true)
+ String output;
+ @Parameter(names = "--columns", description = "columns to extract, in
cf:cq{,cf:cq,...} form")
- String columns;
++ String columns = "";
+ }
+
+ /**
+ * The Mapper class that given a row number, will generate the appropriate
output line.
+ */
+ public static class TTFMapper extends Mapper<Key,Value,NullWritable,Text> {
+ @Override
+ public void map(Key row, Value data, Context context) throws IOException,
InterruptedException {
+ final Key r = row;
+ final Value v = data;
+ Map.Entry<Key,Value> entry = new Map.Entry<Key,Value>() {
+ @Override
+ public Key getKey() {
+ return r;
+ }
+
+ @Override
+ public Value getValue() {
+ return v;
+ }
+
+ @Override
+ public Value setValue(Value value) {
+ return null;
+ }
+ };
+ context.write(NullWritable.get(), new
Text(DefaultFormatter.formatEntry(entry, false)));
+ context.setStatus("Outputed Value");
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws IOException, InterruptedException,
ClassNotFoundException, AccumuloSecurityException {
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" +
System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
+ Opts opts = new Opts();
+ opts.parseArgs(getClass().getName(), args);
+
+ job.setInputFormatClass(AccumuloInputFormat.class);
+ opts.setAccumuloConfigs(job);
+
+ HashSet<Pair<Text,Text>> columnsToFetch = new HashSet<Pair<Text,Text>>();
+ for (String col : opts.columns.split(",")) {
+ int idx = col.indexOf(":");
+ Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
+ Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
+ if (cf.getLength() > 0)
+ columnsToFetch.add(new Pair<Text,Text>(cf, cq));
+ }
+ if (!columnsToFetch.isEmpty())
+ AccumuloInputFormat.fetchColumns(job, columnsToFetch);
+
+ job.setMapperClass(TTFMapper.class);
+ job.setMapOutputKeyClass(NullWritable.class);
+ job.setMapOutputValueClass(Text.class);
+
+ job.setNumReduceTasks(0);
+
+ job.setOutputFormatClass(TextOutputFormat.class);
+ TextOutputFormat.setOutputPath(job, new Path(opts.output));
+
+ job.waitForCompletion(true);
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ /**
+ *
+ * @param args
+ * instanceName zookeepers username password table columns
outputpath
+ * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new
TableToFile(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c175bd31/test/system/auto/simple/examples.py
----------------------------------------------------------------------
diff --cc test/system/auto/simple/examples.py
index 069148d,b22411a..3ba425e
--- a/test/system/auto/simple/examples.py
+++ b/test/system/auto/simple/examples.py
@@@ -41,11 -41,15 +41,11 @@@ class Examples(TestUtilsMixin, unittest
order = 21
def runExample(self, cmd):
- return self.wait(self.runOn(self.masterHost(), [self.accumulo_sh(),]
+ cmd))
+ self.assert_(self.wait(self.runOn(self.masterHost(),
[self.accumulo_sh(),] + cmd)), "example exited with error status.")
- def ignoreExample(self, cmd):
- self.comment(" Ignoring results of command.")
- self.wait(self.runOn(self.masterHost(), [self.accumulo_sh(),] + cmd))
-
- def ashell(self, input):
+ def ashell(self, input, expected = 0):
out, err, code = self.shell(self.masterHost(), input + '\n')
- self.assert_(code == 0)
+ self.assert_(code == expected)
return out
def comment(self, description):
@@@ -58,17 -68,9 +64,16 @@@
self.wait(self.runOn('localhost', cmd))
def runTest(self):
+ examplesJar =
glob.glob(ACCUMULO_HOME+'/lib/accumulo-examples-simple*.jar')[0]
-
+ self.comment("Testing MaxMutation constraint")
+ self.ashell('createtable test_ingest\n'
+ 'constraint -a
org.apache.accumulo.examples.simple.constraints.MaxMutationSize\n')
+ handle = self.runOn('localhost', [self.accumulo_sh(),
'org.apache.accumulo.test.TestIngest', '-u', ROOT, '--rows', '1', '--start',
'0', '--cols', '10000', '-p', ROOT_PASSWORD])
+ out, err = handle.communicate()
+ self.failIf(handle.returncode==0)
+ self.failUnless(err.find("MutationsRejectedException: # constraint
violations : 1") >= 0, "Was able to insert a mutation larger than max size")
+
self.ashell('createtable %s\nsetauths -u %s -s A,B\nquit\n' %(table,
ROOT))
- examplesJar = glob.glob(ACCUMULO_HOME+'/lib/examples-simple*.jar')[0]
-
self.comment("Testing dirlist example (a little)")
self.comment(" ingesting accumulo source")
self.execute(self.accumulo_sh(),
'org.apache.accumulo.examples.simple.dirlist.Ingest',
@@@ -103,156 -105,167 +108,161 @@@
self.comment("Testing bloom filters are fast for missing data")
self.ashell('createtable bloom_test\nconfig -t bloom_test -s
table.bloom.enabled=true\n')
- self.execute(self.accumulo_sh(),
'org.apache.accumulo.examples.simple.client.RandomBatchWriter', '-s', '7',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD,
'bloom_test',
- '1000000', '0', '1000000000', '50', '2000000', '60000',
'3', 'A')
+ self.execute(self.accumulo_sh(),
'org.apache.accumulo.examples.simple.client.RandomBatchWriter', '--seed', '7',
+ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p',
ROOT_PASSWORD, '-t', 'bloom_test',
+ '--num', '1000000', '--min', '0', '--max', '1000000000',
'--size', '50', '--batchMemory', '2M', '--batchLatency', '60s',
+ '--batchThreads', '3')
self.ashell('flush -t bloom_test -w\n')
now = time.time()
- self.execute(self.accumulo_sh(),
'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '-s', '7',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD,
'bloom_test',
- 500, 0, 1000000000, 50, 20, 'A')
+ self.execute(self.accumulo_sh(),
'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '--seed', '7',
+ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p',
ROOT_PASSWORD, '-t', 'bloom_test',
+ '--num', '500', '--min', '0', '--max', '1000000000',
'--size', '50', '--scanThreads', 4)
diff = time.time() - now
now = time.time()
- self.execute(self.accumulo_sh(),
'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '--seed', '8',
- self.executeExpectFail(self.accumulo_sh(),
'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '-s', '8',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD,
'bloom_test',
- 500, 0, 1000000000, 50, 20, 'A')
++ self.executeExpectFail(self.accumulo_sh(),
'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '--seed', '8',
+ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p',
ROOT_PASSWORD, '-t', 'bloom_test',
+ '--num', '500', '--min', '0', '--max', '1000000000',
'--size', '50', '--scanThreads', 4)
diff2 = time.time() - now
self.assert_(diff2 < diff)
self.comment("Creating a sharded index of the accumulo java files")
self.ashell('createtable shard\ncreatetable doc2term\nquit\n')
self.execute('/bin/sh', '-c',
- 'find src -name "*.java" | xargs ./bin/accumulo
org.apache.accumulo.simple.examples.shard.Index -i %s -z %s -t shard -u %s -p
%s --partitions 30' %
- 'find src -name "*.java" | xargs ./bin/accumulo
org.apache.accumulo.examples.simple.shard.Index %s %s shard %s %s 30' %
++ 'find server -name "*.java" | xargs ./bin/accumulo
org.apache.accumulo.examples.simple.shard.Index -i %s -z %s -t shard -u %s -p
%s --partitions 30' %
(INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD))
- self.execute(self.accumulo_sh(),
'org.apache.accumulo.simple.examples.shard.Query',
+ self.execute(self.accumulo_sh(),
'org.apache.accumulo.examples.simple.shard.Query',
- INSTANCE_NAME, ZOOKEEPERS, 'shard', ROOT, ROOT_PASSWORD,
+ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-t', 'shard',
'-u', ROOT, '-p', ROOT_PASSWORD,
'foo', 'bar')
self.comment("Creating a word index of the sharded files")
- self.execute(self.accumulo_sh(),
'org.apache.accumulo.simple.examples.shard.Reverse',
- '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-t', 'shard',
'--doc2Term', 'doc2term', '-u', ROOT, '-p', ROOT_PASSWORD)
+ self.execute(self.accumulo_sh(),
'org.apache.accumulo.examples.simple.shard.Reverse',
- INSTANCE_NAME, ZOOKEEPERS, 'shard', 'doc2term', ROOT,
ROOT_PASSWORD)
++ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '--shardTable',
'shard', '--doc2Term', 'doc2term', '-u', ROOT, '-p', ROOT_PASSWORD)
self.comment("Making 1000 conjunctive queries of 5 random words")
- self.execute(self.accumulo_sh(),
'org.apache.accumulo.simple.examples.shard.ContinuousQuery',
- '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-t', 'shard',
'--doc2Term', 'doc2term', '-u', ROOT, '-p', ROOT_PASSWORD, '--term', 5,
'--count', 1000)
-
- self.execute('hadoop', 'fs', '-rmr', "/tmp/input", "/tmp/files",
"/tmp/splits.txt", "/tmp/failures")
- self.execute('hadoop', 'fs', '-mkdir', "/tmp/input")
+ self.execute(self.accumulo_sh(),
'org.apache.accumulo.examples.simple.shard.ContinuousQuery',
- INSTANCE_NAME, ZOOKEEPERS, 'shard', 'doc2term', ROOT,
ROOT_PASSWORD, 5, 1000)
-
- self.executeIgnoreFail('hadoop', 'fs', '-rmr', "/tmp/input",
"/tmp/files", "/tmp/splits.txt", "/tmp/failures")
- self.execute('hadoop', 'fs', '-mkdir', "/tmp/input")
++ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '--shardTable',
'shard', '--doc2Term', 'doc2term', '-u', ROOT, '-p', ROOT_PASSWORD, '--terms',
5, '--count', 1000)
++ self.executeIgnoreFail('hadoop', 'fs', '-rmr', "tmp/input",
"tmp/files", "tmp/splits.txt", "tmp/failures")
++ self.execute('hadoop', 'fs', '-mkdir', "tmp/input")
self.comment("Starting bulk ingest example")
self.comment(" Creating some test data")
- self.execute(self.accumulo_sh(),
'org.apache.accumulo.simple.examples.mapreduce.bulk.GenerateTestData', 0,
1000000, '/tmp/input/data')
- self.execute(self.accumulo_sh(),
'org.apache.accumulo.simple.examples.mapreduce.bulk.SetupTable',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable')
- self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar,
'org.apache.accumulo.simple.examples.mapreduce.bulk.BulkIngestExample',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable',
'/tmp/input', '/tmp')
- self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar,
'org.apache.accumulo.simple.examples.mapreduce.bulk.VerifyIngest',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable',
0, 1000000)
- self.execute(self.accumulo_sh(),
'org.apache.accumulo.examples.simple.mapreduce.bulk.GenerateTestData', 0,
1000000, '/tmp/input/data')
++ self.execute(self.accumulo_sh(),
'org.apache.accumulo.examples.simple.mapreduce.bulk.GenerateTestData',
'--start-row', 0, '--count', 1000000, '--output', 'tmp/input/data')
+ self.execute(self.accumulo_sh(),
'org.apache.accumulo.examples.simple.mapreduce.bulk.SetupTable',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable')
++ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p',
ROOT_PASSWORD, '-t', 'bulkTable')
+ self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar,
'org.apache.accumulo.examples.simple.mapreduce.bulk.BulkIngestExample',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable',
'/tmp/input', '/tmp')
- self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar,
'org.apache.accumulo.examples.simple.mapreduce.bulk.VerifyIngest',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable',
0, 1000000)
++ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p',
ROOT_PASSWORD, '-t', 'bulkTable', '--inputDir', 'tmp/input', '--workDir',
'tmp')
++ self.execute(self.accumulo_sh(),
'org.apache.accumulo.examples.simple.mapreduce.bulk.VerifyIngest',
++ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p',
ROOT_PASSWORD, '-t', 'bulkTable', '--start-row', 0, '--count', 1000000)
self.wait(self.runOn(self.masterHost(), [
-- 'hadoop', 'fs', '-rmr', "/tmp/tableFile", "/tmp/nines"
++ 'hadoop', 'fs', '-rmr', "tmp/tableFile", "tmp/nines"
]))
self.comment("Running TeraSortIngest for a million rows")
++ self.ashell('createtable sorted\nquit\n')
# 10,000 times smaller than the real terasort
ROWS = 1000*1000
self.wait(self.runOn(self.masterHost(), [
ACCUMULO_HOME+'/bin/tool.sh',
examplesJar,
- 'org.apache.accumulo.simple.examples.mapreduce.TeraSortIngest',
+ 'org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest',
- ROWS,
- 10, 10,
- 78, 78,
- 'sorted',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 4]))
+ '--count', ROWS,
+ '-nk', 10, '-xk', 10,
+ '-nk', 78, '-xk', 78,
+ '-t', 'sorted',
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '--splits', 4]))
self.comment("Looking for '999' in all rows")
++ self.ashell('deletetable sorted\ncreatetable sorted\nquit\n')
self.wait(self.runOn(self.masterHost(), [
ACCUMULO_HOME+'/bin/tool.sh',
examplesJar,
- 'org.apache.accumulo.simple.examples.mapreduce.RegexExample',
+ 'org.apache.accumulo.examples.simple.mapreduce.RegexExample',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 'sorted',
- '.*999.*',
- '.*',
- '.*',
- '.*',
- '/tmp/nines']))
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', 'sorted',
+ '--rowRegex', '.*999.*',
- '/tmp/nines']))
++ 'tmp/nines']))
self.comment("Generating hashes of each row into a new table")
++ self.ashell('deletetable sorted\ncreatetable sorted\nquit\n')
self.wait(self.runOn(self.masterHost(), [
ACCUMULO_HOME+'/bin/tool.sh',
examplesJar,
- 'org.apache.accumulo.simple.examples.mapreduce.RowHash',
+ 'org.apache.accumulo.examples.simple.mapreduce.RowHash',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 'sorted',
- ':',
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', 'sorted',
+ '--column', ':',
'sortedHashed',
]))
self.comment("Exporting the table to HDFS")
++ self.ashell('deletetable sorted\ncreatetable sorted\nquit\n')
self.wait(self.runOn(self.masterHost(), [
ACCUMULO_HOME+'/bin/tool.sh',
examplesJar,
- 'org.apache.accumulo.simple.examples.mapreduce.TableToFile',
+ 'org.apache.accumulo.examples.simple.mapreduce.TableToFile',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 'sortedHashed',
- ',',
- '/tmp/tableFile'
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', 'sorted',
- '--output', '/tmp/tableFile'
++ '--output', 'tmp/tableFile'
]))
self.comment("Running WordCount using Accumulo aggregators")
self.wait(self.runOn(self.masterHost(), [
-- 'hadoop', 'fs', '-rmr', "/tmp/wc"
++ 'hadoop', 'fs', '-rmr', "tmp/wc"
]))
self.wait(self.runOn(self.masterHost(), [
-- 'hadoop', 'fs', '-mkdir', "/tmp/wc"
++ 'hadoop', 'fs', '-mkdir', "tmp/wc"
]))
self.wait(self.runOn(self.masterHost(), [
-- 'hadoop', 'fs', '-copyFromLocal', ACCUMULO_HOME + "/README",
"/tmp/wc/Accumulo.README"
++ 'hadoop', 'fs', '-copyFromLocal', ACCUMULO_HOME + "/README",
"tmp/wc/Accumulo.README"
]))
- self.ashell('createtable wordCount -a
count=org.apache.accumulo.core.iterators.aggregation.StringSummation\nquit\n')
+ self.ashell('createtable wordCount\nsetiter -scan -majc -minc -p 10
-n sum -class
org.apache.accumulo.core.iterators.user.SummingCombiner\n\ncount\n\nSTRING\nquit\n')
self.wait(self.runOn(self.masterHost(), [
ACCUMULO_HOME+'/bin/tool.sh',
examplesJar,
- 'org.apache.accumulo.simple.examples.mapreduce.WordCount',
+ 'org.apache.accumulo.examples.simple.mapreduce.WordCount',
- INSTANCE_NAME,
- ZOOKEEPERS,
- '/tmp/wc',
- 'wctable'
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
- '--input', '/tmp/wc',
++ '-u', ROOT,
++ '-p', ROOT_PASSWORD,
++ '--input', 'tmp/wc',
+ '-t', 'wctable'
]))
self.comment("Inserting data with a batch writer")
-
self.runExample(['org.apache.accumulo.simple.examples.helloworld.InsertWithBatchWriter',
+
self.runExample(['org.apache.accumulo.examples.simple.helloworld.InsertWithBatchWriter',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 'helloBatch'])
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-t', 'helloBatch',
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD])
self.comment("Reading data")
-
self.runExample(['org.apache.accumulo.simple.examples.helloworld.ReadData',
+
self.runExample(['org.apache.accumulo.examples.simple.helloworld.ReadData',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 'helloBatch'])
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-t', 'helloBatch',
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD])
self.comment("Running isolated scans")
-
self.runExample(['org.apache.accumulo.simple.examples.isolation.InterferenceTest',
+
self.runExample(['org.apache.accumulo.examples.simple.isolation.InterferenceTest',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 'itest1',
- 100000,
- 'true'])
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', 'itest1',
+ '--iterations', 100000,
+ '--isolated'])
self.comment("Running scans without isolation")
-
self.runExample(['org.apache.accumulo.simple.examples.isolation.InterferenceTest',
+
self.runExample(['org.apache.accumulo.examples.simple.isolation.InterferenceTest',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 'itest2',
- 100000,
- 'false'])
- self.comment("Inserting data using a map/reduce job")
-
self.runExample(['org.apache.accumulo.examples.simple.helloworld.InsertWithOutputFormat',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 'helloOutputFormat'])
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', 'itest2',
+ '--iterations', 100000])
self.comment("Using some example constraints")
self.ashell('\n'.join([
'createtable testConstraints',
@@@ -262,83 -275,87 +272,84 @@@
'insert r1 cf1 cq1 ABC',
'scan',
'quit'
- ]))
+ ]), 1)
self.comment("Performing some row operations")
-
self.runExample(['org.apache.accumulo.simple.examples.client.RowOperations',
-
self.runExample(['org.apache.accumulo.examples.simple.client.RowOperations',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD])
++
self.runExample(['org.apache.accumulo.examples.simple.client.RowOperations',
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD ])
self.comment("Using the batch writer")
-
self.runExample(['org.apache.accumulo.simple.examples.client.SequentialBatchWriter',
+
self.runExample(['org.apache.accumulo.examples.simple.client.SequentialBatchWriter',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- table,
- min,
- count,
- valueSize,
- memory,
- latency,
- numThreads,
- visibility])
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', table,
+ '--start', min,
+ '--num', count,
+ '--size', valueSize,
+ '--batchMemory', memory,
+ '--batchLatency', latency,
+ '--batchThreads', numThreads,
+ '--vis', visibility])
self.comment("Reading and writing some data")
-
self.runExample(['org.apache.accumulo.simple.examples.client.ReadWriteExample',
+
self.runExample(['org.apache.accumulo.examples.simple.client.ReadWriteExample',
'-i', INSTANCE_NAME,
'-z', ZOOKEEPERS,
'-u', ROOT,
'-p', ROOT_PASSWORD,
- '-s', auths,
- '-t', table,
- '-e',
- '-r',
- '-dbg'])
+ '--auths', auths,
- '-t', table,
- '--createtable',
++ '--table', table,
+ '-c',
+ '--debug'])
self.comment("Deleting some data")
-
self.runExample(['org.apache.accumulo.simple.examples.client.ReadWriteExample',
+
self.runExample(['org.apache.accumulo.examples.simple.client.ReadWriteExample',
'-i', INSTANCE_NAME,
'-z', ZOOKEEPERS,
'-u', ROOT,
'-p', ROOT_PASSWORD,
-- '-s', auths,
-- '-t', table,
++ '-auths', auths,
++ '--table', table,
'-d',
- '-dbg'])
+ '--debug'])
self.comment("Writing some random data with the batch writer")
-
self.runExample(['org.apache.accumulo.simple.examples.client.RandomBatchWriter',
+
self.runExample(['org.apache.accumulo.examples.simple.client.RandomBatchWriter',
- '-s',
- 5,
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- table,
- count,
- min,
- max,
- valueSize,
- memory,
- latency,
- numThreads,
- visibility])
- self.comment("Reading some random data with the batch scanner")
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', table,
++ '--seed','5',
+ '--num', count,
+ '--min', min,
+ '--max', max,
+ '--size', valueSize,
+ '--batchMemory', memory,
+ '--batchLatency', latency,
+ '--batchThreads', numThreads,
+ '--vis', visibility])
+ self.comment("Writing some random data with the batch writer")
-
self.runExample(['org.apache.accumulo.simple.examples.client.RandomBatchScanner',
+
self.runExample(['org.apache.accumulo.examples.simple.client.RandomBatchScanner',
- '-s',
- 5,
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- table,
- count,
- min,
- max,
- valueSize,
- numThreads,
- auths]);
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', table,
++ '--seed','5',
+ '--num', count,
+ '--min', min,
+ '--max', max,
+ '--size', valueSize,
+ '--scanThreads', numThreads,
+ '--auths', auths]);
self.comment("Running an example table operation (Flush)")
- self.runExample(['org.apache.accumulo.simple.examples.client.Flush',
+ self.runExample(['org.apache.accumulo.examples.simple.client.Flush',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- table])
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', table])
self.shutdown_accumulo();
