Author: billie
Date: Fri Sep 7 12:56:20 2012
New Revision: 1382004
URL: http://svn.apache.org/viewvc?rev=1382004&view=rev
Log:
ACCUMULO-700 ACCUMULO-507 changed InputFormat internal configuration
serialization for iterators (now uses IteratorSetting) and ranges (now written
to a file)
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/security/AuthorizationsTest.java
accumulo/trunk/test/system/continuous/ (props changed)
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java?rev=1382004&r1=1382003&r2=1382004&view=diff
==============================================================================
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
Fri Sep 7 12:56:20 2012
@@ -16,6 +16,9 @@
*/
package org.apache.accumulo.core.client;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -28,6 +31,8 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.util.ArgumentChecker;
import org.apache.accumulo.core.util.Pair;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
/**
* Configure an iterator for minc, majc, and/or scan. By default,
IteratorSetting will be configured for scan.
@@ -42,7 +47,7 @@ import org.apache.hadoop.io.Text;
* scanner.addScanIterator(cfg);
* </pre>
*/
-public class IteratorSetting {
+public class IteratorSetting implements Writable {
private int priority;
private String name;
private String iteratorClass;
@@ -185,6 +190,11 @@ public class IteratorSetting {
this(priority, name, iteratorClass.getName());
}
+ public IteratorSetting(DataInput din) throws IOException {
+ this.properties = new HashMap<String,String>();
+ this.readFields(din);
+ }
+
/**
* Add another option to the iterator.
*
@@ -297,4 +307,29 @@ public class IteratorSetting {
}
}
+
+ @Override
+ public void readFields(DataInput din) throws IOException {
+ priority = WritableUtils.readVInt(din);
+ name = WritableUtils.readString(din);
+ iteratorClass = WritableUtils.readString(din);
+ properties.clear();
+ int size = WritableUtils.readVInt(din);
+ while (size > 0) {
+ properties.put(WritableUtils.readString(din),
WritableUtils.readString(din));
+ size--;
+ }
+ }
+
+ @Override
+ public void write(DataOutput dout) throws IOException {
+ WritableUtils.writeVInt(dout, priority);
+ WritableUtils.writeString(dout, name);
+ WritableUtils.writeString(dout, iteratorClass);
+ WritableUtils.writeVInt(dout, properties.size());
+ for (Entry<String,String> e : properties.entrySet()) {
+ WritableUtils.writeString(dout, e.getKey());
+ WritableUtils.writeString(dout, e.getValue());
+ }
+ }
}
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java?rev=1382004&r1=1382003&r2=1382004&view=diff
==============================================================================
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
Fri Sep 7 12:56:20 2012
@@ -23,11 +23,8 @@ import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.io.UnsupportedEncodingException;
import java.math.BigInteger;
import java.net.InetAddress;
-import java.net.URLDecoder;
-import java.net.URLEncoder;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
@@ -141,7 +138,6 @@ public abstract class InputFormatBase<K,
// Used for specifying the iterators to be applied
private static final String ITERATORS = PREFIX + ".iterators";
- private static final String ITERATORS_OPTIONS = PREFIX +
".iterators.options";
private static final String ITERATORS_DELIM = ",";
private static final String READ_OFFLINE = PREFIX + ".read.offline";
@@ -212,7 +208,7 @@ public abstract class InputFormatBase<K,
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
-
+
}
/**
@@ -259,17 +255,24 @@ public abstract class InputFormatBase<K,
*/
public static void setRanges(Configuration conf, Collection<Range> ranges) {
ArgumentChecker.notNull(ranges);
- ArrayList<String> rangeStrings = new ArrayList<String>(ranges.size());
try {
+ FileSystem fs = FileSystem.get(conf);
+ Path file = new Path(fs.getWorkingDirectory(),
conf.get("mapred.job.name") + System.currentTimeMillis() + ".ranges");
+ conf.set(RANGES, file.toString());
+ FSDataOutputStream fos = fs.create(file, false);
+ fs.setPermission(file, new FsPermission(FsAction.ALL, FsAction.NONE,
FsAction.NONE));
+ fs.deleteOnExit(file);
+
+ fos.writeInt(ranges.size());
for (Range r : ranges) {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- r.write(new DataOutputStream(baos));
- rangeStrings.add(new String(Base64.encodeBase64(baos.toByteArray())));
+ r.write(fos);
}
- } catch (IOException ex) {
- throw new IllegalArgumentException("Unable to encode ranges to Base64",
ex);
+ fos.close();
+
+ DistributedCache.addCacheFile(file.toUri(), conf);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to write ranges to file", e);
}
- conf.setStrings(RANGES, rangeStrings.toArray(new String[0]));
}
/**
@@ -381,32 +384,25 @@ public abstract class InputFormatBase<K,
// First check to see if anything has been set already
String iterators = conf.get(ITERATORS);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ String newIter;
+ try {
+ cfg.write(new DataOutputStream(baos));
+ newIter = new String(Base64.encodeBase64(baos.toByteArray()));
+ baos.close();
+ } catch (IOException e) {
+ throw new IllegalArgumentException("unable to serialize
IteratorSetting");
+ }
+
// No iterators specified yet, create a new string
if (iterators == null || iterators.isEmpty()) {
- iterators = new AccumuloIterator(cfg.getPriority(),
cfg.getIteratorClass(), cfg.getName()).toString();
+ iterators = newIter;
} else {
// append the next iterator & reset
- iterators = iterators.concat(ITERATORS_DELIM + new
AccumuloIterator(cfg.getPriority(), cfg.getIteratorClass(),
cfg.getName()).toString());
+ iterators = iterators.concat(ITERATORS_DELIM + newIter);
}
// Store the iterators w/ the job
conf.set(ITERATORS, iterators);
- for (Entry<String,String> entry : cfg.getOptions().entrySet()) {
- if (entry.getValue() == null)
- continue;
-
- String iteratorOptions = conf.get(ITERATORS_OPTIONS);
-
- // No options specified yet, create a new string
- if (iteratorOptions == null || iteratorOptions.isEmpty()) {
- iteratorOptions = new AccumuloIteratorOption(cfg.getName(),
entry.getKey(), entry.getValue()).toString();
- } else {
- // append the next option & reset
- iteratorOptions = iteratorOptions.concat(ITERATORS_DELIM + new
AccumuloIteratorOption(cfg.getName(), entry.getKey(), entry.getValue()));
- }
-
- // Store the options w/ the job
- conf.set(ITERATORS_OPTIONS, iteratorOptions);
- }
}
/**
@@ -541,12 +537,21 @@ public abstract class InputFormatBase<K,
*/
protected static List<Range> getRanges(Configuration conf) throws
IOException {
ArrayList<Range> ranges = new ArrayList<Range>();
- for (String rangeString : conf.getStringCollection(RANGES)) {
- ByteArrayInputStream bais = new
ByteArrayInputStream(Base64.decodeBase64(rangeString.getBytes()));
- Range range = new Range();
- range.readFields(new DataInputStream(bais));
- ranges.add(range);
+ FileSystem fs = FileSystem.get(conf);
+ String rangePath = conf.get(RANGES);
+ if (rangePath == null)
+ return ranges;
+ Path file = new Path(rangePath);
+
+ FSDataInputStream fdis = fs.open(file);
+ int numRanges = fdis.readInt();
+ while (numRanges > 0) {
+ Range r = new Range();
+ r.readFields(fdis);
+ ranges.add(r);
+ numRanges--;
}
+ fdis.close();
return ranges;
}
@@ -617,7 +622,7 @@ public abstract class InputFormatBase<K,
if (!usesLocalIterators(conf)) {
// validate that any scan-time iterators can be loaded by the the
tablet servers
- for (AccumuloIterator iter : getIterators(conf)) {
+ for (IteratorSetting iter : getIterators(conf)) {
if (!c.instanceOperations().testClassLoad(iter.getIteratorClass(),
SortedKeyValueIterator.class.getName()))
throw new AccumuloException("Servers are unable to load " +
iter.getIteratorClass() + " as a " + SortedKeyValueIterator.class.getName());
}
@@ -656,45 +661,26 @@ public abstract class InputFormatBase<K,
* @return a list of iterators
* @see #addIterator(Configuration, IteratorSetting)
*/
- protected static List<AccumuloIterator> getIterators(Configuration conf) {
+ protected static List<IteratorSetting> getIterators(Configuration conf) {
String iterators = conf.get(ITERATORS);
// If no iterators are present, return an empty list
if (iterators == null || iterators.isEmpty())
- return new ArrayList<AccumuloIterator>();
+ return new ArrayList<IteratorSetting>();
// Compose the set of iterators encoded in the job configuration
StringTokenizer tokens = new StringTokenizer(conf.get(ITERATORS),
ITERATORS_DELIM);
- List<AccumuloIterator> list = new ArrayList<AccumuloIterator>();
- while (tokens.hasMoreTokens()) {
- String itstring = tokens.nextToken();
- list.add(new AccumuloIterator(itstring));
- }
- return list;
- }
-
- /**
- * Gets a list of the iterator options specified on this configuration.
- *
- * @param conf
- * the Hadoop configuration object
- * @return a list of iterator options
- * @see #addIterator(Configuration, IteratorSetting)
- */
- protected static List<AccumuloIteratorOption>
getIteratorOptions(Configuration conf) {
- String iteratorOptions = conf.get(ITERATORS_OPTIONS);
-
- // If no options are present, return an empty list
- if (iteratorOptions == null || iteratorOptions.isEmpty())
- return new ArrayList<AccumuloIteratorOption>();
-
- // Compose the set of options encoded in the job configuration
- StringTokenizer tokens = new StringTokenizer(conf.get(ITERATORS_OPTIONS),
ITERATORS_DELIM);
- List<AccumuloIteratorOption> list = new
ArrayList<AccumuloIteratorOption>();
- while (tokens.hasMoreTokens()) {
- String optionString = tokens.nextToken();
- list.add(new AccumuloIteratorOption(optionString));
+ List<IteratorSetting> list = new ArrayList<IteratorSetting>();
+ try {
+ while (tokens.hasMoreTokens()) {
+ String itstring = tokens.nextToken();
+ ByteArrayInputStream bais = new
ByteArrayInputStream(Base64.decodeBase64(itstring.getBytes()));
+ list.add(new IteratorSetting(new DataInputStream(bais)));
+ bais.close();
+ }
+ } catch (IOException e) {
+ throw new IllegalArgumentException("couldn't decode iterator settings");
}
return list;
}
@@ -714,18 +700,9 @@ public abstract class InputFormatBase<K,
* @throws AccumuloException
*/
protected void setupIterators(Configuration conf, Scanner scanner) throws
AccumuloException {
- List<AccumuloIterator> iterators = getIterators(conf);
- List<AccumuloIteratorOption> options = getIteratorOptions(conf);
-
- Map<String,IteratorSetting> scanIterators = new
HashMap<String,IteratorSetting>();
- for (AccumuloIterator iterator : iterators) {
- scanIterators.put(iterator.getIteratorName(), new
IteratorSetting(iterator.getPriority(), iterator.getIteratorName(),
iterator.getIteratorClass()));
- }
- for (AccumuloIteratorOption option : options) {
- scanIterators.get(option.iteratorName).addOption(option.getKey(),
option.getValue());
- }
- for (AccumuloIterator iterator : iterators) {
- scanner.addScanIterator(scanIterators.get(iterator.getIteratorName()));
+ List<IteratorSetting> iterators = getIterators(conf);
+ for (IteratorSetting iterator : iterators) {
+ scanner.addScanIterator(iterator);
}
}
@@ -1142,101 +1119,4 @@ public abstract class InputFormatBase<K,
out.writeUTF(locations[i]);
}
}
-
- /**
- * The Class IteratorSetting. Encapsulates specifics for an Accumulo
iterator's name & priority.
- */
- static class AccumuloIterator {
-
- private static final String FIELD_SEP = ":";
-
- private int priority;
- private String iteratorClass;
- private String iteratorName;
-
- public AccumuloIterator(int priority, String iteratorClass, String
iteratorName) {
- this.priority = priority;
- this.iteratorClass = iteratorClass;
- this.iteratorName = iteratorName;
- }
-
- // Parses out a setting given an string supplied from an earlier
toString() call
- public AccumuloIterator(String iteratorSetting) {
- // Parse the string to expand the iterator
- StringTokenizer tokenizer = new StringTokenizer(iteratorSetting,
FIELD_SEP);
- priority = Integer.parseInt(tokenizer.nextToken());
- iteratorClass = tokenizer.nextToken();
- iteratorName = tokenizer.nextToken();
- }
-
- public int getPriority() {
- return priority;
- }
-
- public String getIteratorClass() {
- return iteratorClass;
- }
-
- public String getIteratorName() {
- return iteratorName;
- }
-
- @Override
- public String toString() {
- return new String(priority + FIELD_SEP + iteratorClass + FIELD_SEP +
iteratorName);
- }
-
- }
-
- /**
- * The Class AccumuloIteratorOption. Encapsulates specifics for an Accumulo
iterator's optional configuration details - associated via the iteratorName.
- */
- static class AccumuloIteratorOption {
- private static final String FIELD_SEP = ":";
-
- private String iteratorName;
- private String key;
- private String value;
-
- public AccumuloIteratorOption(String iteratorName, String key, String
value) {
- this.iteratorName = iteratorName;
- this.key = key;
- this.value = value;
- }
-
- // Parses out an option given a string supplied from an earlier toString()
call
- public AccumuloIteratorOption(String iteratorOption) {
- StringTokenizer tokenizer = new StringTokenizer(iteratorOption,
FIELD_SEP);
- this.iteratorName = tokenizer.nextToken();
- try {
- this.key = URLDecoder.decode(tokenizer.nextToken(), "UTF-8");
- this.value = URLDecoder.decode(tokenizer.nextToken(), "UTF-8");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- }
- }
-
- public String getIteratorName() {
- return iteratorName;
- }
-
- public String getKey() {
- return key;
- }
-
- public String getValue() {
- return value;
- }
-
- @Override
- public String toString() {
- try {
- return new String(iteratorName + FIELD_SEP + URLEncoder.encode(key,
"UTF-8") + FIELD_SEP + URLEncoder.encode(value, "UTF-8"));
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- }
- }
-
- }
-
}
Modified:
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java?rev=1382004&r1=1382003&r2=1382004&view=diff
==============================================================================
---
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
(original)
+++
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
Fri Sep 7 12:56:20 2012
@@ -19,23 +19,26 @@ package org.apache.accumulo.core.client.
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
import java.io.IOException;
+import java.util.HashSet;
import java.util.List;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.IteratorSetting;
-import
org.apache.accumulo.core.client.mapreduce.InputFormatBase.AccumuloIterator;
-import
org.apache.accumulo.core.client.mapreduce.InputFormatBase.AccumuloIteratorOption;
import
org.apache.accumulo.core.client.mapreduce.InputFormatBase.RangeInputSplit;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.user.RegExFilter;
import org.apache.accumulo.core.iterators.user.WholeRowIterator;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.ContextFactory;
+import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
@@ -89,15 +92,19 @@ public class AccumuloInputFormatTest {
/**
* Check that the iterator configuration is getting stored in the Job conf
correctly.
+ * @throws IOException
*/
@Test
- public void testSetIterator() {
+ public void testSetIterator() throws IOException {
JobContext job = ContextFactory.createJobContext();
- AccumuloInputFormat.addIterator(job.getConfiguration(), new
IteratorSetting(1, "WholeRow",
"org.apache.accumulo.core.iterators.WholeRowIterator"));
+ IteratorSetting is = new IteratorSetting(1, "WholeRow",
"org.apache.accumulo.core.iterators.WholeRowIterator");
+ AccumuloInputFormat.addIterator(job.getConfiguration(), is);
Configuration conf = job.getConfiguration();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ is.write(new DataOutputStream(baos));
String iterators = conf.get("AccumuloInputFormat.iterators");
-
assertEquals("1:org.apache.accumulo.core.iterators.WholeRowIterator:WholeRow",
iterators);
+ assertEquals(new String(Base64.encodeBase64(baos.toByteArray())),
iterators);
}
@Test
@@ -111,35 +118,31 @@ public class AccumuloInputFormatTest {
iter.addOption("junk", "\0omg:!\\xyzzy");
AccumuloInputFormat.addIterator(job.getConfiguration(), iter);
- List<AccumuloIterator> list =
AccumuloInputFormat.getIterators(job.getConfiguration());
+ List<IteratorSetting> list =
AccumuloInputFormat.getIterators(job.getConfiguration());
// Check the list size
assertTrue(list.size() == 3);
// Walk the list and make sure our settings are correct
- AccumuloIterator setting = list.get(0);
+ IteratorSetting setting = list.get(0);
assertEquals(1, setting.getPriority());
assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator",
setting.getIteratorClass());
- assertEquals("WholeRow", setting.getIteratorName());
+ assertEquals("WholeRow", setting.getName());
+ assertEquals(0, setting.getOptions().size());
setting = list.get(1);
assertEquals(2, setting.getPriority());
assertEquals("org.apache.accumulo.core.iterators.VersioningIterator",
setting.getIteratorClass());
- assertEquals("Versions", setting.getIteratorName());
+ assertEquals("Versions", setting.getName());
+ assertEquals(0, setting.getOptions().size());
setting = list.get(2);
assertEquals(3, setting.getPriority());
assertEquals("org.apache.accumulo.core.iterators.CountingIterator",
setting.getIteratorClass());
- assertEquals("Count", setting.getIteratorName());
-
- List<AccumuloIteratorOption> iteratorOptions =
AccumuloInputFormat.getIteratorOptions(job.getConfiguration());
- assertEquals(2, iteratorOptions.size());
- assertEquals("Count", iteratorOptions.get(0).getIteratorName());
- assertEquals("Count", iteratorOptions.get(1).getIteratorName());
- assertEquals("v1", iteratorOptions.get(0).getKey());
- assertEquals("1", iteratorOptions.get(0).getValue());
- assertEquals("junk", iteratorOptions.get(1).getKey());
- assertEquals("\0omg:!\\xyzzy", iteratorOptions.get(1).getValue());
+ assertEquals("Count", setting.getName());
+ assertEquals(2, setting.getOptions().size());
+ assertEquals("1", setting.getOptions().get("v1"));
+ assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk"));
}
/**
@@ -157,25 +160,22 @@ public class AccumuloInputFormatTest {
Job job = new Job();
AccumuloInputFormat.addIterator(job.getConfiguration(), someSetting);
- final String rawConfigOpt = new AccumuloIteratorOption("iterator", key,
value).toString();
-
- assertEquals(rawConfigOpt,
job.getConfiguration().get("AccumuloInputFormat.iterators.options"));
-
- List<AccumuloIteratorOption> opts =
AccumuloInputFormat.getIteratorOptions(job.getConfiguration());
- assertEquals(1, opts.size());
- assertEquals(opts.get(0).getKey(), key);
- assertEquals(opts.get(0).getValue(), value);
+ List<IteratorSetting> list =
AccumuloInputFormat.getIterators(job.getConfiguration());
+ assertEquals(1, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
someSetting.addOption(key + "2", value);
someSetting.setPriority(2);
someSetting.setName("it2");
AccumuloInputFormat.addIterator(job.getConfiguration(), someSetting);
- opts = AccumuloInputFormat.getIteratorOptions(job.getConfiguration());
- assertEquals(3, opts.size());
- for (AccumuloIteratorOption opt : opts) {
- assertEquals(opt.getKey().substring(0, key.length()), key);
- assertEquals(opt.getValue(), value);
- }
+ list = AccumuloInputFormat.getIterators(job.getConfiguration());
+ assertEquals(2, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+ assertEquals(2, list.get(1).getOptions().size());
+ assertEquals(list.get(1).getOptions().get(key), value);
+ assertEquals(list.get(1).getOptions().get(key + "2"), value);
}
/**
@@ -189,26 +189,26 @@ public class AccumuloInputFormatTest {
AccumuloInputFormat.addIterator(job.getConfiguration(), new
IteratorSetting(2, "Versions",
"org.apache.accumulo.core.iterators.VersioningIterator"));
AccumuloInputFormat.addIterator(job.getConfiguration(), new
IteratorSetting(3, "Count",
"org.apache.accumulo.core.iterators.CountingIterator"));
- List<AccumuloIterator> list =
AccumuloInputFormat.getIterators(job.getConfiguration());
+ List<IteratorSetting> list =
AccumuloInputFormat.getIterators(job.getConfiguration());
// Check the list size
assertTrue(list.size() == 3);
// Walk the list and make sure our settings are correct
- AccumuloIterator setting = list.get(0);
+ IteratorSetting setting = list.get(0);
assertEquals(1, setting.getPriority());
assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator",
setting.getIteratorClass());
- assertEquals("WholeRow", setting.getIteratorName());
+ assertEquals("WholeRow", setting.getName());
setting = list.get(1);
assertEquals(2, setting.getPriority());
assertEquals("org.apache.accumulo.core.iterators.VersioningIterator",
setting.getIteratorClass());
- assertEquals("Versions", setting.getIteratorName());
+ assertEquals("Versions", setting.getName());
setting = list.get(2);
assertEquals(3, setting.getPriority());
assertEquals("org.apache.accumulo.core.iterators.CountingIterator",
setting.getIteratorClass());
- assertEquals("Count", setting.getIteratorName());
+ assertEquals("Count", setting.getName());
}
@@ -222,19 +222,22 @@ public class AccumuloInputFormatTest {
RegExFilter.setRegexs(is, regex, null, null, null, false);
AccumuloInputFormat.addIterator(job.getConfiguration(), is);
-
assertTrue(regex.equals(AccumuloInputFormat.getIterators(job.getConfiguration()).get(0).getIteratorName()));
+
assertTrue(regex.equals(AccumuloInputFormat.getIterators(job.getConfiguration()).get(0).getName()));
}
static class TestMapper extends Mapper<Key,Value,Key,Value> {
Key key = null;
+ int first = 0;
int count = 0;
@Override
protected void map(Key k, Value v, Context context) throws IOException,
InterruptedException {
if (key != null)
assertEquals(key.getRow().toString(), new String(v.get()));
- assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
- assertEquals(new String(v.get()), String.format("%09x", count));
+ else
+ first = Integer.parseInt(k.getRow().toString(), 16) - 1;
+ assertEquals(k.getRow(), new Text(String.format("%09x", first + count +
1)));
+ assertEquals(new String(v.get()), String.format("%09x", first + count));
key = new Key(k);
count++;
}
@@ -259,10 +262,14 @@ public class AccumuloInputFormatTest {
job.setNumReduceTasks(0);
AccumuloInputFormat.setInputInfo(job.getConfiguration(), "root",
"".getBytes(), "testtable", new Authorizations());
AccumuloInputFormat.setMockInstance(job.getConfiguration(),
"testmapinstance");
+ HashSet<Range> ranges = new HashSet<Range>();
+ ranges.add(new Range("000000000", "000000010"));
+ ranges.add(new Range("000000100", "000000110"));
+ AccumuloInputFormat.setRanges(job.getConfiguration(), ranges);
AccumuloInputFormat input = new AccumuloInputFormat();
List<InputSplit> splits = input.getSplits(job);
- assertEquals(splits.size(), 1);
+ assertEquals(splits.size(), 2);
TestMapper mapper = (TestMapper) job.getMapperClass().newInstance();
for (InputSplit split : splits) {
@@ -271,6 +278,7 @@ public class AccumuloInputFormatTest {
Mapper<Key,Value,Key,Value>.Context context =
ContextFactory.createMapContext(mapper, tac, reader, null, split);
reader.initialize(split, context);
mapper.run(context);
+ assertEquals(mapper.count, 16);
}
}
@@ -302,5 +310,6 @@ public class AccumuloInputFormatTest {
while (rr.nextKeyValue()) {
mapper.map(rr.getCurrentKey(), rr.getCurrentValue(),
(TestMapper.Context) context);
}
+ assertEquals(mapper.count, 100);
}
}
Modified:
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/security/AuthorizationsTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/security/AuthorizationsTest.java?rev=1382004&r1=1382003&r2=1382004&view=diff
==============================================================================
---
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/security/AuthorizationsTest.java
(original)
+++
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/security/AuthorizationsTest.java
Fri Sep 7 12:56:20 2012
@@ -35,8 +35,6 @@ public class AuthorizationsTest {
byte[] array = a.getAuthorizationsArray();
Authorizations b = new Authorizations(array);
assertEquals(a, b);
-
- System.out.println(b);
}
}
Propchange: accumulo/trunk/test/system/continuous/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Fri Sep 7 12:56:20 2012
@@ -1,5 +1,5 @@
continuous-env.sh
-
walkers.txt
-
ingesters.txt
+batch_walkers.txt
+scanners.txt
