svn commit: r1435013 - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/client/ core/src/main/java/org/apache/accumulo/core/client/mapreduce/ core/src/main/java/org/apache/accumulo/core
Author: ctubbsii
Date: Fri Jan 18 03:18:29 2013
New Revision: 1435013
URL: http://svn.apache.org/viewvc?rev=1435013&view=rev
Log:
ACCUMULO-955 Made BatchWriterConfig Writable, so it can be stored in a job's
configuration in a human-readable way. Updated AccumuloOutputFormat to use it,
and
added unit tests for ACCUMULO-706 and ACCUMULO-955. Added an additional check
for a reasonable minimum maxMemory value.
Added:
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteCommand.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InsertCommand.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java?rev=1435013&r1=1435012&r2=1435013&view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
Fri Jan 18 03:18:29 2013
@@ -16,24 +16,43 @@
*/
package org.apache.accumulo.core.client;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.StringUtils;
+
/**
* This object holds configuration settings used to instantiate a {@link
BatchWriter}
*/
-public class BatchWriterConfig {
- private long maxMemory = 50 * 1024 * 1024;
- private long maxLatency = 12;
- private long timeout = Long.MAX_VALUE;
- private int maxWriteThreads = 3;
+public class BatchWriterConfig implements Writable {
+
+ private static final Long DEFAULT_MAX_MEMORY = 50 * 1024 * 1024l;
+ private Long maxMemory = null;
+
+ private static final Long DEFAULT_MAX_LATENCY = 2 * 60 * 1000l;
+ private Long maxLatency = null;
+
+ private static final Long DEFAULT_TIMEOUT = Long.MAX_VALUE;
+ private Long timeout = null;
+
+ private static final Integer DEFAULT_MAX_WRITE_THREADS = 3;
+ private Integer maxWriteThreads = null;
/**
*
* @param maxMemory
- * size in bytes of the maximum memory to batch before writing.
Defaults to 50M.
+ * size in bytes of the maximum memory to batch before writing.
Minimum 1K. Defaults to 50M.
*/
public BatchWriterConfig setMaxMemory(long maxMemory) {
+if (maxMemory < 1024)
+ throw new IllegalArgumentException("Max memory is too low at " +
maxMemory + ". Minimum 1K.");
this.maxMemory = maxMemory;
return this;
}
@@ -93,18 +112,76 @@ public class BatchWriterConfig {
}
public long getMaxMemory() {
-return maxMemory;
+return maxMemory != null ? maxMemory : DEFAULT_MAX_MEMORY;
}
public long getMaxLatency(TimeUnit timeUnit) {
-return timeUnit.convert(maxLatency, TimeUnit.MILLISECONDS);
+return timeUnit.convert(maxLatency != null ? maxLatency :
DEFAULT_MAX_LATENCY, TimeUnit.MILLISECONDS);
}
public long getTimeout(TimeUnit timeUnit) {
-return timeUnit.convert(timeout, TimeUnit.MILLISECONDS);
+return timeUnit.convert(timeout != null ? timeout : DEFAULT_TIMEOUT,
TimeUnit.MILLISECONDS);
}
public int getMaxWriteThreads() {
-return maxWriteThreads;
+return maxWriteThreads != null ? maxWriteThreads :
DEFAULT_MAX_WRITE_THREADS;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+// write this out in a human-readable way
+ArrayList fields = new ArrayList();
+if (maxMemory != null)
+ addField(fields, "maxMemory", maxMemory);
+if (maxLatency != null)
+ addField(fields, "maxLatency", maxLatency);
+if (maxWriteThreads != null)
+ addField(fields, "maxWriteThreads", maxWriteThreads);
+if (timeout != null)
+ addField(fields, "timeout", timeout);
+String output = StringUtils.join(",", fields);
+
+byte[] bytes = output.getBytes(Charset.forName("UTF-8"));
+byte[] len = String.format("%6s#", Integer.toString(bytes.length,
36)).getBytes("UTF-8");
+if (len.length != 7)
svn commit: r1434983 - /accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/MergeCommand.java
Author: billie
Date: Fri Jan 18 00:18:06 2013
New Revision: 1434983
URL: http://svn.apache.org/viewvc?rev=1434983&view=rev
Log:
ACCUMULO-947 documented exclusivity of begin row option for merge shell command
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/MergeCommand.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/MergeCommand.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/MergeCommand.java?rev=1434983&r1=1434982&r2=1434983&view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/MergeCommand.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/MergeCommand.java
Fri Jan 18 00:18:06 2013
@@ -92,7 +92,9 @@ public class MergeCommand extends Comman
verboseOpt = new Option("v", "verbose", false, "verbose output during
merge");
sizeOpt = new Option("s", "size", true, "merge tablets to the given size
over the entire table");
forceOpt = new Option("f", "force", false, "merge small tablets to large
tablets, even if it goes over the given size");
-o.addOption(OptUtil.startRowOpt());
+Option startRowOpt = OptUtil.startRowOpt();
+startRowOpt.setDescription("begin row (NOT inclusive)");
+o.addOption(startRowOpt);
o.addOption(OptUtil.endRowOpt());
o.addOption(OptUtil.tableOpt("table to be merged"));
o.addOption(verboseOpt);
svn commit: r1434955 - in /accumulo/trunk/core/src: main/java/org/apache/accumulo/core/iterators/user/ test/java/org/apache/accumulo/core/iterators/user/
Author: kturner
Date: Thu Jan 17 23:17:20 2013
New Revision: 1434955
URL: http://svn.apache.org/viewvc?rev=1434955&view=rev
Log:
ACCUMULO-956 generlaized transforming iterator, added some sanity checks to it,
added some more unit test, added some static config methods
Added:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
- copied, changed from r1434936,
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java
- copied, changed from r1434936,
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
Removed:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
Copied:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
(from r1434936,
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java)
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java?p2=accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java&p1=accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java&r1=1434936&r2=1434955&rev=1434955&view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
Thu Jan 17 23:17:20 2013
@@ -23,7 +23,10 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Map;
+import java.util.NoSuchElementException;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.PartialKey;
@@ -40,12 +43,13 @@ import org.apache.accumulo.core.security
import org.apache.accumulo.core.security.VisibilityParseException;
import org.apache.accumulo.core.util.BadArgumentException;
import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.collections.BufferOverflowException;
import org.apache.commons.collections.map.LRUMap;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
/**
- * The KeyTransformingIterator allows portions of a key (except for the row)
to be transformed. This iterator handles the details that come with modifying
keys
+ * The TransformingIterator allows portions of a key (except for the row) to
be transformed. This iterator handles the details that come with modifying keys
* (i.e., that the sort order could change). In order to do so, however, the
iterator must put all keys sharing the same prefix in memory. Prefix is defined
as
* the parts of the key that are not modified by this iterator. That is, if
the iterator modifies column qualifier and timestamp, then the prefix is row and
* column family. In that case, the iterator must load all column qualifiers
for each row/column family pair into memory. Given this constraint, care must be
@@ -70,8 +74,11 @@ import org.apache.log4j.Logger;
* major and minor compactions. It should also be noted that this iterator
implements the security filtering rather than relying on a follow-on iterator
to do
* it so that we ensure the test is performed.
*/
-abstract public class KeyTransformingIterator extends WrappingIterator
implements OptionDescriber {
+abstract public class TransformingIterator extends WrappingIterator implements
OptionDescriber {
public static final String AUTH_OPT = "authorizations";
+ public static final String MAX_BUFFER_SIZE_OPT = "maxBufferSize";
+ private static final long DEFAULT_MAX_BUFFER_SIZE = 1000;
+
protected Logger log = Logger.getLogger(getClass());
protected ArrayList> keys = new ArrayList>();
@@ -84,6 +91,7 @@ abstract public class KeyTransformingIte
private VisibilityEvaluator ve = null;
private LRUMap visibleCache = null;
private LRUMap parsedVisibilitiesCache = null;
+ private long maxBufferSize;
private static Comparator> keyComparator = new
Comparator>() {
@Override
@@ -92,7 +100,7 @@ abstract public class KeyTransformingIte
}
};
- public KeyTransformingIterator() {}
+ public TransformingIterator() {}
@Override
public void init(SortedKeyValueIterator source,
Map options, IteratorEnvironment env) throws IOException {
@@ -101,11 +109,
svn commit: r1434928 - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/client/ core/src/main/java/org/apache/accumulo/core/client/impl/ core/src/main/java/org/apache/accumulo/core/data
Author: ctubbsii
Date: Thu Jan 17 21:46:32 2013
New Revision: 1434928
URL: http://svn.apache.org/viewvc?rev=1434928&view=rev
Log:
ACCUMULO-973 Remove trivial warnings about imports, and added missing
hashCode() implementations for objects that override equals(), to be nice to
users.
ACCUMULO-971 Fix some malformed javadoc return tags, and errors linking to
hidden (protected) methods (I don't know why javadoc cares about these).
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InterpreterCommand.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/TStore.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MiniAccumuloConfig.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java?rev=1434928&r1=1434927&r2=1434928&view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
Thu Jan 17 21:46:32 2013
@@ -31,9 +31,8 @@ public class BatchWriterConfig {
*
* @param maxMemory
* size in bytes of the maximum memory to batch before writing.
Defaults to 50M.
- * @return
*/
-
+
public BatchWriterConfig setMaxMemory(long maxMemory) {
this.maxMemory = maxMemory;
return this;
@@ -47,11 +46,11 @@ public class BatchWriterConfig {
* Determines how maxLatency will be interpreted.
* @return this to allow chaining of set methods
*/
-
+
public BatchWriterConfig setMaxLatency(long maxLatency, TimeUnit timeUnit) {
if (maxLatency < 0)
throw new IllegalArgumentException("Negative max latency not allowed " +
maxLatency);
-
+
if (maxLatency == 0)
this.maxLatency = Long.MAX_VALUE;
else
@@ -67,7 +66,7 @@ public class BatchWriterConfig {
* @param timeUnit
* @return this to allow chaining of set methods
*/
-
+
public BatchWriterConfig setTimeout(long timeout, TimeUnit timeUnit) {
if (timeout < 0)
throw new IllegalArgumentException("Negative timeout not allowed " +
timeout);
@@ -84,15 +83,15 @@ public class BatchWriterConfig {
* the maximum number of threads to use for writing data to the
tablet servers. Defaults to 3.
* @return this to allow chaining of set methods
*/
-
+
public BatchWriterConfig setMaxWriteThreads(int maxWriteThreads) {
if (maxWriteThreads <= 0)
throw new IllegalArgumentException("Max threads must be positive " +
maxWriteThreads);
-
+
this.maxWriteThreads = maxWriteThreads;
return this;
}
-
+
public long getMaxMemory() {
return maxMemory;
}
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java?rev=1434928&r1=1434927&r2=1434928&view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
Thu Jan 17 21:46:32 2013
@@ -116,7 +116,6 @@ class OfflineIterator implements Iterato
private ArrayList> readers;
/**
- * @param offlineScanner
* @param instance
* @param credentials
* @param authorizations
Modified:
accumulo/trunk/core/src/main/java/org/apache/accu
svn commit: r1434921 [2/7] - in /accumulo/trunk: ./ conf/examples/1GB/native-standalone/ conf/examples/1GB/standalone/ conf/examples/2GB/native-standalone/ conf/examples/2GB/standalone/ conf/examples/
Added:
accumulo/trunk/proxy/src/main/java/org/apache/accumulo/proxy/thrift/AccumuloException.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/proxy/src/main/java/org/apache/accumulo/proxy/thrift/AccumuloException.java?rev=1434921&view=auto
==
---
accumulo/trunk/proxy/src/main/java/org/apache/accumulo/proxy/thrift/AccumuloException.java
(added)
+++
accumulo/trunk/proxy/src/main/java/org/apache/accumulo/proxy/thrift/AccumuloException.java
Thu Jan 17 21:22:35 2013
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Autogenerated by Thrift Compiler (0.9.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.accumulo.proxy.thrift;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.protocol.TProtocolException;
+import org.apache.thrift.EncodingUtils;
+import org.apache.thrift.TException;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings("all") public class AccumuloException extends TException
implements org.apache.thrift.TBase, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new
org.apache.thrift.protocol.TStruct("AccumuloException");
+
+ private static final org.apache.thrift.protocol.TField MSG_FIELD_DESC = new
org.apache.thrift.protocol.TField("msg",
org.apache.thrift.protocol.TType.STRING, (short)1);
+
+ private static final Map, SchemeFactory> schemes =
new HashMap, SchemeFactory>();
+ static {
+schemes.put(StandardScheme.class, new
AccumuloExceptionStandardSchemeFactory());
+schemes.put(TupleScheme.class, new AccumuloExceptionTupleSchemeFactory());
+ }
+
+ public String msg; // required
+
+ /** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
+ @SuppressWarnings("all") public enum _Fields implements
org.apache.thrift.TFieldIdEnum {
+MSG((short)1, "msg");
+
+private static final Map byName = new HashMap();
+
+static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+byName.put(field.getFieldName(), field);
+ }
+}
+
+/**
+ * Find the _Fields constant that matches fieldId, or null if its not
found.
+ */
+public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+case 1: // MSG
+ return MSG;
+default:
+ return null;
+ }
+}
+
+/**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " +
fieldId + " doesn't exist!");
+ return fields;
+}
+
+/**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+public static _Fields findByName(String name) {
+ return byName.get(name);
+}
+
+private final short _thriftId;
+private final String _fieldName;
+
+_Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+}
+
+public short getThriftFieldId() {
+ return _thriftId;
+}
+
+public String getFieldName() {
+ return _fieldName;
+}
+ }
+
+ // isset id assignments
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData>
metaDataMap;
+ static {
+Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new
EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_
svn commit: r1434881 - in /accumulo/trunk/core/src: main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIter
Author: kturner
Date: Thu Jan 17 20:01:44 2013
New Revision: 1434881
URL: http://svn.apache.org/viewvc?rev=1434881&view=rev
Log:
ACCUMULO-956 fixed some issues and added unit test for transforming iterator
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java?rev=1434881&r1=1434880&r2=1434881&view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
Thu Jan 17 20:01:44 2013
@@ -279,26 +279,40 @@ abstract public class KeyTransformingIte
* @return {@code true} if the key is visible or iterator is not scanning,
and {@code false} if not
*/
protected boolean canSee(Key key) {
-// Ensure that the visibility (which could have been transformed) parses.
+// Ensure that the visibility (which could have been transformed) parses.
Must always do this check, even if visibility is not evaluated.
ByteSequence visibility = key.getColumnVisibilityData();
-ColumnVisibility colVis = (ColumnVisibility)
parsedVisibilitiesCache.get(visibility);
-if (colVis == null) {
+ColumnVisibility colVis = null;
+Boolean parsed = (Boolean) parsedVisibilitiesCache.get(visibility);
+if (parsed == null) {
try {
colVis = new ColumnVisibility(visibility.toArray());
+parsedVisibilitiesCache.put(visibility, Boolean.TRUE);
} catch (BadArgumentException e) {
-log.error("Transformation produced an invalid visibility: " +
visibility);
-throw e;
+log.error("Parse error after transformation : " + visibility);
+parsedVisibilitiesCache.put(visibility, Boolean.FALSE);
+if (scanning) {
+ return false;
+} else {
+ throw e;
+}
}
+} else if (!parsed) {
+ if (scanning)
+return false;
+ else
+throw new IllegalStateException();
}
Boolean visible = canSeeColumnFamily(key);
-if (!scanning || !visible || ve == null || visibleCache == null)
+if (!scanning || !visible || ve == null || visibleCache == null ||
visibility.length() == 0)
return visible;
visible = (Boolean) visibleCache.get(visibility);
if (visible == null) {
try {
+if (colVis == null)
+ colVis = new ColumnVisibility(visibility.toArray());
visible = ve.evaluate(colVis);
visibleCache.put(visibility, visible);
} catch (VisibilityParseException e) {
Modified:
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java?rev=1434881&r1=1434880&r2=1434881&view=diff
==
---
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
(original)
+++
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
Thu Jan 17 20:01:44 2013
@@ -16,18 +16,24 @@
*/
package org.apache.accumulo.core.iterators.user;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.SortedMap;
import java.util.TreeMap;
+import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
@@ -43,9 +49,9 @@ import org.apache.accumulo.core.data.Par
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.WrappingIterator;
-import org.apache.accumulo.core.iterators.IteratorUtil.Itera
svn commit: r1434768 - in /accumulo/trunk/core/src: main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIter
Author: kturner
Date: Thu Jan 17 16:33:47 2013
New Revision: 1434768
URL: http://svn.apache.org/viewvc?rev=1434768&view=rev
Log:
ACCUMULO-956 formatted source code
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java?rev=1434768&r1=1434767&r2=1434768&view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
Thu Jan 17 16:33:47 2013
@@ -45,49 +45,30 @@ import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
/**
- * The KeyTransformingIterator allows portions of a key (except for the row)
- * to be transformed. This iterator handles the details that come with
modifying
- * keys (i.e., that the sort order could change). In order to do so, however,
- * the iterator must put all keys sharing the same prefix in memory. Prefix
- * is defined as the parts of the key that are not modified by this iterator.
- * That is, if the iterator modifies column qualifier and timestamp, then the
- * prefix is row and column family. In that case, the iterator must load all
- * column qualifiers for each row/column family pair into memory. Given this
- * constraint, care must be taken by users of this iterator to ensure it is
- * not run in such a way that will overrun memory in a tablet server.
+ * The KeyTransformingIterator allows portions of a key (except for the row)
to be transformed. This iterator handles the details that come with modifying
keys
+ * (i.e., that the sort order could change). In order to do so, however, the
iterator must put all keys sharing the same prefix in memory. Prefix is defined
as
+ * the parts of the key that are not modified by this iterator. That is, if
the iterator modifies column qualifier and timestamp, then the prefix is row and
+ * column family. In that case, the iterator must load all column qualifiers
for each row/column family pair into memory. Given this constraint, care must be
+ * taken by users of this iterator to ensure it is not run in such a way that
will overrun memory in a tablet server.
*
- * If the implementing iterator is transforming column families, then it
- * must also override {@link #untransformColumnFamilies(Collection)} to handle
- * the case when column families are fetched at scan time. The fetched column
- * families will/must be in the transformed space, and the untransformed column
- * families need to be passed to this iterator's source. If it is not possible
- * to write a reverse transformation (e.g., the column family transformation
- * depends on the row value or something like that), then the iterator must
- * not fetch specific column families (or only fetch column families that are
- * known to not transform at all).
+ * If the implementing iterator is transforming column families, then it must
also override {@link #untransformColumnFamilies(Collection)} to handle the case
+ * when column families are fetched at scan time. The fetched column families
will/must be in the transformed space, and the untransformed column families
need
+ * to be passed to this iterator's source. If it is not possible to write a
reverse transformation (e.g., the column family transformation depends on the
row
+ * value or something like that), then the iterator must not fetch specific
column families (or only fetch column families that are known to not transform
at
+ * all).
*
- * If the implementing iterator is transforming column visibilities, then
- * users must be careful NOT to fetch column qualifiers from the scanner.
- * The reason for this is due to ACCUMULO-??? (insert issue number).
+ * If the implementing iterator is transforming column visibilities, then
users must be careful NOT to fetch column qualifiers from the scanner. The
reason for
+ * this is due to ACCUMULO-??? (insert issue number).
*
- * If the implementing iterator is transforming column visibilities, then the
- * user should be sure to supply authorizations via the {@link #AUTH_OPT}
- * iterator option (note that this is only necessary for scan scope iterators).
- * The supplied authorizations should be in the transformed space, but the
- * authorizations supplied to the scanner should be in the untransformed
- * space. That is, if the iterator transforms A to 1, B to 2, C to 3, etc,
- * then the auths supplied when the scanner is constructed should be A,B,C,...
- * and the au
svn commit: r1434762 - in /accumulo/trunk/core/src: main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIter
Author: kturner
Date: Thu Jan 17 16:28:48 2013
New Revision: 1434762
URL: http://svn.apache.org/viewvc?rev=1434762&view=rev
Log:
ACCUMULO-956 checkin of patch from Brain Loss
Added:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
Added:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java?rev=1434762&view=auto
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
(added)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
Thu Jan 17 16:28:48 2013
@@ -0,0 +1,599 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Map;
+
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.OptionDescriber;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.WrappingIterator;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.VisibilityEvaluator;
+import org.apache.accumulo.core.security.VisibilityParseException;
+import org.apache.accumulo.core.util.BadArgumentException;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.collections.map.LRUMap;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+/**
+ * The KeyTransformingIterator allows portions of a key (except for the row)
+ * to be transformed. This iterator handles the details that come with
modifying
+ * keys (i.e., that the sort order could change). In order to do so, however,
+ * the iterator must put all keys sharing the same prefix in memory. Prefix
+ * is defined as the parts of the key that are not modified by this iterator.
+ * That is, if the iterator modifies column qualifier and timestamp, then the
+ * prefix is row and column family. In that case, the iterator must load all
+ * column qualifiers for each row/column family pair into memory. Given this
+ * constraint, care must be taken by users of this iterator to ensure it is
+ * not run in such a way that will overrun memory in a tablet server.
+ *
+ * If the implementing iterator is transforming column families, then it
+ * must also override {@link #untransformColumnFamilies(Collection)} to handle
+ * the case when column families are fetched at scan time. The fetched column
+ * families will/must be in the transformed space, and the untransformed column
+ * families need to be passed to this iterator's source. If it is not possible
+ * to write a reverse transformation (e.g., the column family transformation
+ * depends on the row value or something like that), then the iterator must
+ * not fetch specific column families (or only fetch column families that are
+ * known to not transform at all).
+ *
+ * If the implementing iterator is transforming column visibilities, then
+ * users must be careful NOT to fetch column qualifiers from the scanner.
+ * The reason for this is due to ACCUMULO-??? (insert issue number).
+ *
+ * If the implementing iterator is transforming column visibilities, then the
+ * user should be sure to supply authorizations via the {@link #AUTH_OPT}
+ * iterator option (note t
svn commit: r1434751 - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/iterators/ core/src/main/java/org/apache/accumulo/core/iterators/user/ core/src/test/java/org/apache/accumulo/cor
Author: billie
Date: Thu Jan 17 16:05:12 2013
New Revision: 1434751
URL: http://svn.apache.org/viewvc?rev=1434751&view=rev
Log:
ACCUMULO-846 cleaned up options validation
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/Filter.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/FirstEntryInRowIterator.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/OptionDescriber.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/SortedKeyIterator.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/AgeOffFilter.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/ColumnAgeOffFilter.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/LargeRowFilter.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TimestampFilter.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java
accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/combiner/StatsCombiner.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java?rev=1434751&r1=1434750&r2=1434751&view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java
Thu Jan 17 16:05:12 2013
@@ -190,7 +190,7 @@ public class AggregatingIterator impleme
for (Entry entry : options.entrySet()) {
String classname = entry.getValue();
if (classname == null)
-return false;
+throw new IllegalArgumentException("classname null");
Class clazz;
try {
clazz = AccumuloVFSClassLoader.loadClass(classname, Aggregator.class);
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java?rev=1434751&r1=1434750&r2=1434751&view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
Thu Jan 17 16:05:12 2013
@@ -35,13 +35,13 @@ import org.apache.accumulo.core.iterator
import org.apache.log4j.Logger;
/**
- * A SortedKeyValueIterator that combines the Values for different versions
(timestamps) of a Key into a single Value. Combiner will replace one or more
versions of a Key
- * and their Values with the most recent Key and a Value which is the result
of the reduce method.
+ * A SortedKeyValueIterator that combines the Values for different versions
(timestamps) of a Key into a single Value. Combiner will replace one or more
+ * versions of a Key and their Values with the most recent Key and a Value
which is the result of the reduce method.
*
* Subclasses must implement a reduce method: {@code public Value reduce(Key
key, Iterator iter)}.
*
- * This reduce method will be passed the most recent Key and an iterator over
the Values for all non-deleted versions of that Key.
- * A combiner will not combine keys that differ by more than the timestamp.
+ * This reduce method will be passed the most recent Key and an iterator over
the Values for all non-deleted versions of that Key. A combiner will not combine
+ * keys that differ by more than the timestamp.
*/
public abstract class Combiner extends WrappingIterator implements
OptionDescriber {
static final Logger log = Logger.getLogger(Combiner.class);
@@ -254,20 +254,24 @@ public abstract class Combiner extends W
@Override
public boolean validateOptions(Map options) {
if (options.containsKey(ALL_OPTION)) {
- combineAllColumns = Boolean.parseBoolean(options.get(ALL_OPTION));
+ try {
+combineAllColumns = B
svn commit: r1434696 - in /accumulo/trunk/core/src: main/java/org/apache/accumulo/core/file/rfile/RFile.java test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
Author: kturner
Date: Thu Jan 17 14:45:57 2013
New Revision: 1434696
URL: http://svn.apache.org/viewvc?rev=1434696&view=rev
Log:
ACCUMULO-962 fixed issue where rfile would read more data than it was supposed
to. Added unit test that reproduced the bug.
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java?rev=1434696&r1=1434695&r2=1434696&view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
Thu Jan 17 14:45:57 2013
@@ -642,6 +642,7 @@ public class RFile {
private void _seek(Range range) throws IOException {
this.range = range;
+ this.checkRange = true;
if (blockCount == 0) {
// its an empty file
Modified:
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java?rev=1434696&r1=1434695&r2=1434696&view=diff
==
---
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
(original)
+++
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
Thu Jan 17 14:45:57 2013
@@ -28,6 +28,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Random;
@@ -1442,6 +1443,63 @@ public class RFileTest {
trf.closeReader();
}
+ @Test
+ public void testReseekUnconsumed() throws Exception {
+TestRFile trf = new TestRFile();
+
+trf.openWriter();
+
+for (int i = 0; i < 2500; i++) {
+ trf.writer.append(nk(nf("r_", i), "cf1", "cq1", "L1", 42), nv("foo" +
i));
+}
+
+trf.closeWriter();
+trf.openReader();
+
+Set cfs = Collections.emptySet();
+
+Random rand = new Random();
+
+for (int count = 0; count < 100; count++) {
+
+ int start = rand.nextInt(2300);
+ Range range = new Range(nk(nf("r_", start), "cf1", "cq1", "L1", 42),
nk(nf("r_", start + 100), "cf1", "cq1", "L1", 42));
+
+ trf.reader.seek(range, cfs, false);
+
+ int numToScan = rand.nextInt(100);
+
+ for (int j = 0; j < numToScan; j++) {
+assertTrue(trf.reader.hasTop());
+assertEquals(nk(nf("r_", start + j), "cf1", "cq1", "L1", 42),
trf.reader.getTopKey());
+trf.reader.next();
+ }
+
+ assertTrue(trf.reader.hasTop());
+ assertEquals(nk(nf("r_", start + numToScan), "cf1", "cq1", "L1", 42),
trf.reader.getTopKey());
+
+ // seek a little forward from the last range and read a few keys within
the unconsumed portion of the last range
+
+ int start2 = start + numToScan + rand.nextInt(3);
+ int end2 = start2 + rand.nextInt(3);
+
+ range = new Range(nk(nf("r_", start2), "cf1", "cq1", "L1", 42),
nk(nf("r_", end2), "cf1", "cq1", "L1", 42));
+ trf.reader.seek(range, cfs, false);
+
+ for (int j = start2; j <= end2; j++) {
+assertTrue(trf.reader.hasTop());
+assertEquals(nk(nf("r_", j), "cf1", "cq1", "L1", 42),
trf.reader.getTopKey());
+trf.reader.next();
+ }
+
+ assertFalse(trf.reader.hasTop());
+
+}
+
+trf.closeReader();
+ }
+
+
@Test(expected = NullPointerException.class)
public void testMissingUnreleasedVersions() throws Exception {
runVersionTest(5);
