svn commit: r1325933 - in /accumulo/trunk: ./ core/ core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java core/src/main/java/org/apache/accumulo/core/iterators/system/SynchronizedIte
Author: kturner
Date: Fri Apr 13 20:15:31 2012
New Revision: 1325933
URL: http://svn.apache.org/viewvc?rev=1325933view=rev
Log:
ACCUMULO-533 merged from 1.4
Added:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/system/SynchronizedIterator.java
- copied unchanged from r1325930,
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/system/SynchronizedIterator.java
Modified:
accumulo/trunk/ (props changed)
accumulo/trunk/core/ (props changed)
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
accumulo/trunk/server/ (props changed)
Propchange: accumulo/trunk/
--
Merged /accumulo/branches/1.4/src:r1325859-1325930
Merged /accumulo/branches/1.4:r1325858-1325930
Propchange: accumulo/trunk/core/
--
Merged /accumulo/branches/1.4/core:r1325858-1325930
Merged /accumulo/branches/1.4/src/core:r1325859-1325930
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java?rev=1325933r1=1325932r2=1325933view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
Fri Apr 13 20:15:31 2012
@@ -36,6 +36,7 @@ import org.apache.accumulo.core.data.Key
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.thrift.IterInfo;
import org.apache.accumulo.core.iterators.conf.PerColumnIteratorConfig;
+import org.apache.accumulo.core.iterators.system.SynchronizedIterator;
import org.apache.accumulo.core.iterators.user.VersioningIterator;
import org.apache.accumulo.start.classloader.AccumuloClassLoader;
import org.apache.hadoop.io.Writable;
@@ -213,7 +214,8 @@ public class IteratorUtil {
@SuppressWarnings(unchecked)
public static K extends WritableComparable?,V extends Writable
SortedKeyValueIteratorK,V loadIterators(SortedKeyValueIteratorK,V source,
CollectionIterInfo iters, MapString,MapString,String iterOpts,
IteratorEnvironment env, boolean useAccumuloClassLoader) throws IOException {
-SortedKeyValueIteratorK,V prev = source;
+// wrap the source in a SynchronizedIterator in case any of the additional
configured iterators want to use threading
+SortedKeyValueIteratorK,V prev = new SynchronizedIteratorK,V(source);
try {
for (IterInfo iterInfo : iters) {
Propchange: accumulo/trunk/server/
--
Merged /accumulo/branches/1.4/server:r1325858-1325930
Merged /accumulo/branches/1.4/src/server:r1325859-1325930
svn commit: r1337131 - /accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
Author: kturner
Date: Fri May 11 12:47:09 2012
New Revision: 1337131
URL: http://svn.apache.org/viewvc?rev=1337131view=rev
Log:
ACCUMULO-590 fixed NPE in Table load balancer
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java?rev=1337131r1=1337130r2=1337131view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
Fri May 11 12:47:09 2012
@@ -32,7 +32,6 @@ import org.apache.accumulo.core.conf.Pro
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
import org.apache.accumulo.server.client.HdfsZooInstance;
-import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.master.state.TabletMigration;
import org.apache.accumulo.server.security.SecurityConstants;
@@ -44,11 +43,6 @@ public class TableLoadBalancer extends T
private static final Logger log = Logger.getLogger(TableLoadBalancer.class);
MapString,TabletBalancer perTableBalancers = new
HashMapString,TabletBalancer();
- ServerConfiguration config;
-
- public void init(ServerConfiguration config) {
-this.config = config;
- }
private TabletBalancer constructNewBalancerForTable(String clazzName, String
table) throws Exception {
Class? extends TabletBalancer clazz =
AccumuloClassLoader.loadClass(clazzName, TabletBalancer.class);
@@ -57,7 +51,7 @@ public class TableLoadBalancer extends T
}
protected String getLoadBalancerClassNameForTable(String table) {
-return
config.getTableConfiguration(table).get(Property.TABLE_LOAD_BALANCER);
+return
configuration.getTableConfiguration(table).get(Property.TABLE_LOAD_BALANCER);
}
protected TabletBalancer getBalancerForTable(String table) {
svn commit: r1337287 - in /accumulo/branches/1.4: docs/ src/ src/core/ src/core/src/main/java/org/apache/accumulo/core/client/admin/ src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/ s
Author: kturner Date: Fri May 11 17:11:55 2012 New Revision: 1337287 URL: http://svn.apache.org/viewvc?rev=1337287view=rev Log: ACCUMULO-591 clear tablet location cache before computing input splits (merged from trunk) Modified: accumulo/branches/1.4/docs/config.html accumulo/branches/1.4/src/ (props changed) accumulo/branches/1.4/src/core/ (props changed) accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java accumulo/branches/1.4/src/server/ (props changed) Modified: accumulo/branches/1.4/docs/config.html URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/docs/config.html?rev=1337287r1=1337286r2=1337287view=diff == --- accumulo/branches/1.4/docs/config.html (original) +++ accumulo/branches/1.4/docs/config.html Fri May 11 17:11:55 2012 @@ -202,7 +202,7 @@ $HADOOP_HOME/lib/[^.].*.jar, tr tdmaster.port.client/td tdba href='#PORT'port/a/b/td -tdyes but requires restart/td +tdyes but requires restart of the master/td tdpre/pre/td tdThe port used for handling client connections on the master/td /tr @@ -324,7 +324,7 @@ $HADOOP_HOME/lib/[^.].*.jar, tr tdtserver.compaction.major.concurrent.max/td tdba href='#COUNT'count/a/b/td -tdyes but requires restart/td +tdyes but requires restart of the tserver/td tdpre3/pre/td tdThe maximum number of concurrent major compactions for a tablet server/td /tr @@ -338,7 +338,7 @@ $HADOOP_HOME/lib/[^.].*.jar, tr tdtserver.compaction.major.thread.files.open.max/td tdba href='#COUNT'count/a/b/td -tdyes but requires restart/td +tdyes but requires restart of the tserver/td tdpre10/pre/td tdMax number of files a major compaction thread can open at once. /td /tr @@ -380,7 +380,7 @@ $HADOOP_HOME/lib/[^.].*.jar, tr tdtserver.logger.count/td tdba href='#COUNT'count/a/b/td -tdyes but requires restart/td +tdyes but requires restart of the tserver/td tdpre2/pre/td tdThe number of loggers that each tablet server should use./td /tr @@ -422,7 +422,7 @@ $HADOOP_HOME/lib/[^.].*.jar, tr tdtserver.memory.maps.native.enabled/td tdba href='#BOOLEAN'boolean/a/b/td -tdyes but requires restart/td +tdyes but requires restart of the tserver/td tdpretrue/pre/td tdAn in-memory data store for accumulo implemented in c++ that increases the amount of data accumulo can hold in memory and avoids Java GC pauses./td /tr @@ -457,7 +457,7 @@ $HADOOP_HOME/lib/[^.].*.jar, tr class='highlight' tdtserver.port.client/td tdba href='#PORT'port/a/b/td -tdyes but requires restart/td +tdyes but requires restart of the tserver/td tdpre9997/pre/td tdThe port used for handling client connections on the tablet servers/td /tr @@ -478,7 +478,7 @@ $HADOOP_HOME/lib/[^.].*.jar, tr tdtserver.scan.files.open.max/td tdba href='#COUNT'count/a/b/td -tdyes but requires restart/td +tdyes but requires restart of the tserver/td tdpre100/pre/td tdMaximum total map files that all tablets in a tablet server can open for scans. /td /tr @@ -558,7 +558,7 @@ $HADOOP_HOME/lib/[^.].*.jar, tr tdlogger.port.client/td tdba href='#PORT'port/a/b/td -tdyes but requires restart/td +tdyes but requires restart of the logger/td tdpre11224/pre/td tdThe port used for write-ahead logger services/td /tr @@ -617,7 +617,7 @@ $HADOOP_HOME/lib/[^.].*.jar, tr class='highlight' tdgc.port.client/td tdba href='#PORT'port/a/b/td -tdyes but requires restart/td +tdyes but requires restart of the gc/td tdpre50091/pre/td tdThe listening port for the garbage collector's monitor service/td /tr Propchange: accumulo/branches/1.4/src/ -- Merged /accumulo/trunk:r1337210 Propchange: accumulo/branches/1.4/src/core/ -- Merged /accumulo/trunk/core:r1337210 Modified: accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java?rev=1337287r1=1337286r2=1337287view=diff == --- accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java (original) +++ accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java Fri May 11 17:11:55 2012 @@ -850,6 +850,8 @@ public class
svn commit: r1341596 - /accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
Author: kturner
Date: Tue May 22 19:32:51 2012
New Revision: 1341596
URL: http://svn.apache.org/viewvc?rev=1341596view=rev
Log:
ACCUMULO-348 parallelized adding splits to a table
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java?rev=1341596r1=1341595r2=1341596view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
Tue May 22 19:32:51 2012
@@ -32,6 +32,11 @@ import java.util.Set;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
@@ -301,6 +306,62 @@ public class TableOperationsImpl extends
}
}
+ private static class SplitEnv {
+private String tableName;
+private String tableId;
+private ExecutorService executor;
+private CountDownLatch latch;
+private AtomicReferenceException exception;
+
+SplitEnv(String tableName, String tableId, ExecutorService executor,
CountDownLatch latch, AtomicReferenceException exception) {
+ this.tableName = tableName;
+ this.tableId = tableId;
+ this.executor = executor;
+ this.latch = latch;
+ this.exception = exception;
+}
+ }
+
+ private class SplitTask implements Runnable {
+
+private ListText splits;
+private SplitEnv env;
+
+SplitTask(SplitEnv env, ListText splits) {
+ this.env = env;
+ this.splits = splits;
+}
+
+@Override
+public void run() {
+ try {
+if (env.exception.get() != null)
+ return;
+
+if (splits.size() = 2) {
+ addSplits(env.tableName, new TreeSetText(splits), env.tableId);
+ for (int i = 0; i splits.size(); i++)
+env.latch.countDown();
+ return;
+}
+
+int mid = splits.size() / 2;
+
+// split the middle split point to ensure that child task split
different tablets and can therefore
+// run in parallel
+addSplits(env.tableName, new TreeSetText(splits.subList(mid, mid +
1)), env.tableId);
+env.latch.countDown();
+
+env.executor.submit(new SplitTask(env, splits.subList(0, mid)));
+env.executor.submit(new SplitTask(env, splits.subList(mid + 1,
splits.size(;
+
+ } catch (Exception e) {
+env.exception.compareAndSet(null, e);
+ }
+}
+
+ }
+
/**
* @param tableName
* the name of the table
@@ -315,7 +376,40 @@ public class TableOperationsImpl extends
*/
public void addSplits(String tableName, SortedSetText partitionKeys)
throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
String tableId = Tables.getTableId(instance, tableName);
-addSplits(tableName, partitionKeys, tableId);
+
+ListText splits = new ArrayListText(partitionKeys);
+// should be sorted because we copied from a sorted set, but that makes
assumptions about
+// how the copy was done so resort to be sure.
+Collections.sort(splits);
+
+CountDownLatch latch = new CountDownLatch(splits.size());
+AtomicReferenceException exception = new
AtomicReferenceException(null);
+
+ExecutorService executor = Executors.newFixedThreadPool(16);
+try {
+ executor.submit(new SplitTask(new SplitEnv(tableName, tableId, executor,
latch, exception), splits));
+
+ while (!latch.await(100, TimeUnit.MILLISECONDS)) {
+if (exception.get() != null) {
+ executor.shutdownNow();
+ Exception excep = exception.get();
+ if (excep instanceof TableNotFoundException)
+throw (TableNotFoundException) excep;
+ else if (excep instanceof AccumuloException)
+throw (AccumuloException) excep;
+ else if (excep instanceof AccumuloSecurityException)
+throw (AccumuloSecurityException) excep;
+ else if (excep instanceof RuntimeException)
+throw (RuntimeException) excep;
+ else
+throw new RuntimeException(excep);
+}
+ }
+} catch (InterruptedException e) {
+ throw new RuntimeException(e);
+} finally {
+ executor.shutdown();
+}
}
private void addSplits
svn commit: r1343790 - in /accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client: ./ impl/ mock/
Author: kturner
Date: Tue May 29 15:50:07 2012
New Revision: 1343790
URL: http://svn.apache.org/viewvc?rev=1343790view=rev
Log:
ACCUMULO-580 roll back changes to add batch size to batch scanner
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/IsolatedScanner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/Scanner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerImpl.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockScanner.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java?rev=1343790r1=1343789r2=1343790view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
Tue May 29 15:50:07 2012
@@ -25,6 +25,7 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.impl.ScannerOptions;
import org.apache.accumulo.core.client.mock.IteratorAdapter;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
@@ -52,7 +53,9 @@ import org.apache.hadoop.io.Text;
* the source scanner (which will execute server side) and to the client side
scanner (which will execute client side).
*/
public class ClientSideIteratorScanner extends ScannerOptions implements
Scanner {
+ private int size;
private int timeOut;
+
private Range range;
private boolean isolated = false;
@@ -131,9 +134,9 @@ public class ClientSideIteratorScanner e
*/
public ClientSideIteratorScanner(Scanner scanner) {
smi = new ScannerTranslator(scanner);
-setRange(scanner.getRange());
-setBatchSize(scanner.getBatchSize());
-setTimeOut(scanner.getTimeOut());
+this.range = new Range((Key) null, (Key) null);
+this.size = Constants.SCAN_BATCH_SIZE;
+this.timeOut = Integer.MAX_VALUE;
}
/**
@@ -147,7 +150,7 @@ public class ClientSideIteratorScanner e
@Override
public IteratorEntryKey,Value iterator() {
-smi.scanner.setBatchSize(getBatchSize());
+smi.scanner.setBatchSize(size);
smi.scanner.setTimeOut(timeOut);
if (isolated)
smi.scanner.enableIsolation();
@@ -225,6 +228,16 @@ public class ClientSideIteratorScanner e
}
@Override
+ public void setBatchSize(int size) {
+this.size = size;
+ }
+
+ @Override
+ public int getBatchSize() {
+return size;
+ }
+
+ @Override
public void enableIsolation() {
this.isolated = true;
}
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/IsolatedScanner.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/IsolatedScanner.java?rev=1343790r1=1343789r2=1343790view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/IsolatedScanner.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/IsolatedScanner.java
Tue May 29 15:50:07 2012
@@ -20,6 +20,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map.Entry;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.impl.IsolationException;
import org.apache.accumulo.core.client.impl.ScannerOptions;
import org.apache.accumulo.core.data.ByteSequence;
@@ -209,6 +210,7 @@ public class IsolatedScanner extends Sca
private Scanner scanner;
private Range range;
private int timeOut;
+ private int batchSize;
private RowBufferFactory bufferFactory;
public IsolatedScanner(Scanner scanner) {
@@ -217,15 +219,15 @@ public class IsolatedScanner extends Sca
public IsolatedScanner(Scanner scanner, RowBufferFactory bufferFactory) {
this.scanner = scanner;
-setRange(scanner.getRange());
-setBatchSize(scanner.getBatchSize());
-setTimeOut(scanner.getTimeOut());
+this.range = new Range();
+this.timeOut = Integer.MAX_VALUE;
+this.batchSize = Constants.SCAN_BATCH_SIZE;
this.bufferFactory
svn commit: r1343890 - in /accumulo/branches/1.4: docs/config.html src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
Author: kturner
Date: Tue May 29 18:15:40 2012
New Revision: 1343890
URL: http://svn.apache.org/viewvc?rev=1343890view=rev
Log:
ACCUMULO-509 set default recovery walog replication to 2
Modified:
accumulo/branches/1.4/docs/config.html
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
Modified: accumulo/branches/1.4/docs/config.html
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/docs/config.html?rev=1343890r1=1343889r2=1343890view=diff
==
--- accumulo/branches/1.4/docs/config.html (original)
+++ accumulo/branches/1.4/docs/config.html Tue May 29 18:15:40 2012
@@ -573,7 +573,7 @@ $HADOOP_HOME/lib/[^.].*.jar,
tdlogger.recovery.file.replication/td
tdba href='#COUNT'count/a/b/td
tdyes/td
-tdpre1/pre/td
+tdpre2/pre/td
tdWhen a logger puts a WALOG into HDFS, it will use this as the
replication factor./td
/tr
tr class='highlight'
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1343890r1=1343889r2=1343890view=diff
==
---
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
(original)
+++
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
Tue May 29 18:15:40 2012
@@ -172,7 +172,7 @@ public enum Property {
When enabled the logger will monitor file systems and kill itself when
one switches from rw to ro. This is usually and indication that Linux has
detected a bad disk.),
LOGGER_SORT_BUFFER_SIZE(logger.sort.buffer.size, 200M,
PropertyType.MEMORY,
The amount of memory to use when sorting logs during recovery. Only
used when *not* sorting logs with map/reduce.),
- LOGGER_RECOVERY_FILE_REPLICATION(logger.recovery.file.replication, 1,
PropertyType.COUNT,
+ LOGGER_RECOVERY_FILE_REPLICATION(logger.recovery.file.replication, 2,
PropertyType.COUNT,
When a logger puts a WALOG into HDFS, it will use this as the
replication factor.),
LOGGER_MINTHREADS(logger.server.threads.minimum, 2, PropertyType.COUNT,
The miniumum number of threads to use to handle incoming requests.),
LOGGER_THREADCHECK(logger.server.threadcheck.time, 1s,
PropertyType.TIMEDURATION, The time between adjustments of the server thread
pool.),
svn commit: r1343942 - /accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
Author: kturner
Date: Tue May 29 19:31:57 2012
New Revision: 1343942
URL: http://svn.apache.org/viewvc?rev=1343942view=rev
Log:
ACCUMULO-487 Made batch scanner iterator fail when batch scanner closed
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java?rev=1343942r1=1343941r2=1343942view=diff
==
---
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
(original)
+++
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
Tue May 29 19:31:57 2012
@@ -178,7 +178,7 @@ public class TabletServerBatchReaderIter
// don't have one cached, try to cache one and return success
try {
-while (nextEntry == null fatalException == null)
+while (nextEntry == null fatalException == null
!queryThreadPool.isShutdown())
nextEntry = resultsQueue.poll(1, TimeUnit.SECONDS);
if (fatalException != null)
@@ -187,6 +187,9 @@ public class TabletServerBatchReaderIter
else
throw new RuntimeException(fatalException);
+if (queryThreadPool.isShutdown())
+ throw new RuntimeException(scanner closed);
+
return nextEntry.getKey() != null nextEntry.getValue() != null;
} catch (InterruptedException e) {
throw new RuntimeException(e);
svn commit: r1343945 - in /accumulo/trunk: ./ core/ core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java server/ src/
Author: kturner
Date: Tue May 29 19:37:33 2012
New Revision: 1343945
URL: http://svn.apache.org/viewvc?rev=1343945view=rev
Log:
ACCUMULO-487 Made batch scanner iterator fail when batch scanner closed
Modified:
accumulo/trunk/ (props changed)
accumulo/trunk/core/ (props changed)
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
accumulo/trunk/server/ (props changed)
accumulo/trunk/src/ (props changed)
Propchange: accumulo/trunk/
--
Merged /accumulo/branches/1.4/src:r1343899-1343942
Merged /accumulo/branches/1.4:r1343898-1343942
Propchange: accumulo/trunk/core/
--
Merged /accumulo/branches/1.4/core:r1343898-1343942
Merged /accumulo/branches/1.4/src/core:r1343899-1343942
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java?rev=1343945r1=1343944r2=1343945view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
Tue May 29 19:37:33 2012
@@ -178,7 +178,7 @@ public class TabletServerBatchReaderIter
// don't have one cached, try to cache one and return success
try {
-while (nextEntry == null fatalException == null)
+while (nextEntry == null fatalException == null
!queryThreadPool.isShutdown())
nextEntry = resultsQueue.poll(1, TimeUnit.SECONDS);
if (fatalException != null)
@@ -187,6 +187,9 @@ public class TabletServerBatchReaderIter
else
throw new RuntimeException(fatalException);
+if (queryThreadPool.isShutdown())
+ throw new RuntimeException(scanner closed);
+
return nextEntry.getKey() != null nextEntry.getValue() != null;
} catch (InterruptedException e) {
throw new RuntimeException(e);
Propchange: accumulo/trunk/server/
--
Merged /accumulo/branches/1.4/server:r1343898-1343942
Merged /accumulo/branches/1.4/src/server:r1343899-1343942
Propchange: accumulo/trunk/src/
--
Merged /accumulo/branches/1.4/src:r1343898-1343942
Merged /accumulo/branches/1.4/src/src:r1343899-1343942
svn commit: r1344015 - in /accumulo/trunk/server/src/main/java/org/apache/accumulo/server: tabletserver/Compactor.java tabletserver/MinorCompactor.java test/functional/BadIteratorMincTest.java
Author: kturner
Date: Tue May 29 21:40:45 2012
New Revision: 1344015
URL: http://svn.apache.org/viewvc?rev=1344015view=rev
Log:
ACCUMULO-58 stopped attempting failed minor compaction when table deleted
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/BadIteratorMincTest.java
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java?rev=1344015r1=1344014r2=1344015view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
Tue May 29 21:40:45 2012
@@ -79,7 +79,7 @@ public class Compactor implements Callab
private CompactionEnv env;
private Configuration conf;
private FileSystem fs;
- private KeyExtent extent;
+ protected KeyExtent extent;
private ListIteratorSetting iterators;
Compactor(Configuration conf, FileSystem fs, MapString,DataFileValue
files, InMemoryMap imm, String outputFile, boolean propogateDeletes,
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java?rev=1344015r1=1344014r2=1344015view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java
Tue May 29 21:40:45 2012
@@ -22,10 +22,13 @@ import java.util.Map;
import java.util.Random;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.impl.Tables;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
-import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.master.state.tables.TableState;
import org.apache.accumulo.core.util.MetadataTable.DataFileValue;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.accumulo.server.problems.ProblemReport;
import org.apache.accumulo.server.problems.ProblemReports;
@@ -64,6 +67,15 @@ public class MinorCompactor extends Comp
});
}
+ private boolean isTableDeleting() {
+try {
+ return Tables.getTableState(HdfsZooInstance.getInstance(),
extent.getTableId().toString()) == TableState.DELETING;
+} catch (Exception e) {
+ log.warn(Failed to determine if table + extent.getTableId() + was
deleting , e);
+ return false; // can not get positive confirmation that its deleting.
+}
+ }
+
@Override
public CompactionStats call() {
log.debug(Begin minor compaction + getOutputFile() + + getExtent());
@@ -75,7 +87,6 @@ public class MinorCompactor extends Comp
boolean reportedProblem = false;
do {
-
try {
CompactionStats ret = super.call();
@@ -117,6 +128,9 @@ public class MinorCompactor extends Comp
log.warn(Failed to delete failed MinC file + getOutputFile() +
+ e.getMessage());
}
+ if (isTableDeleting())
+return new CompactionStats(0, 0);
+
} while (true);
}
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/BadIteratorMincTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/BadIteratorMincTest.java?rev=1344015r1=1344014r2=1344015view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/BadIteratorMincTest.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/BadIteratorMincTest.java
Tue May 29 21:40:45 2012
@@ -95,6 +95,26 @@ public class BadIteratorMincTest extends
if (count != 1)
throw new Exception(Did not see expected # entries + count);
+
+// now try putting bad iterator back and deleting the table
+getConnector().tableOperations().setProperty(foo,
Property.TABLE_ITERATOR_PREFIX.getKey() + minc.badi, 30, +
BadIterator.class.getName());
+bw = getConnector().createBatchWriter(foo, 100, 6l, 2);
+m = new
svn commit: r1344302 - /accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java
Author: kturner
Date: Wed May 30 15:14:25 2012
New Revision: 1344302
URL: http://svn.apache.org/viewvc?rev=1344302view=rev
Log:
ACCUMULO-601 bail when an unknownhostexception is seen when trying to connect
to master, no longer retries
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java?rev=1344302r1=1344301r2=1344302view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java
Wed May 30 15:14:25 2012
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.core.client.impl;
+import java.net.UnknownHostException;
import java.util.List;
import org.apache.accumulo.core.client.AccumuloException;
@@ -34,7 +35,7 @@ import org.apache.thrift.transport.TTran
public class MasterClient {
private static final Logger log = Logger.getLogger(MasterClient.class);
- public static MasterClientService.Iface getConnectionWithRetry(Instance
instance) throws TTransportException {
+ public static MasterClientService.Iface getConnectionWithRetry(Instance
instance) {
ArgumentChecker.notNull(instance);
while (true) {
@@ -64,6 +65,10 @@ public class MasterClient {
instance.getConfiguration());
return client;
} catch (TTransportException tte) {
+ if (tte.getCause().getClass().equals(UnknownHostException.class)) {
+// do not expect to recover from this
+throw new RuntimeException(tte);
+ }
log.debug(Failed to connect to master= + master + portHint= +
portHint + , will retry... , tte);
return null;
}
svn commit: r1344358 - /accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
Author: kturner
Date: Wed May 30 17:23:41 2012
New Revision: 1344358
URL: http://svn.apache.org/viewvc?rev=1344358view=rev
Log:
ACCUMULO-601 made the master advertise an IP addr in zookeeper
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1344358r1=1344357r2=1344358view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
Wed May 30 17:23:41 2012
@@ -2157,7 +2157,8 @@ public class Master implements LiveTServ
};
long current = System.currentTimeMillis();
final long waitTime =
getSystemConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
-final String masterClientAddress = hostname + : +
getSystemConfiguration().getPort(Property.MASTER_CLIENTPORT);
+final String masterClientAddress =
org.apache.accumulo.core.util.AddressUtil.toString(new
InetSocketAddress(hostname, getSystemConfiguration().getPort(
+Property.MASTER_CLIENTPORT)));
boolean locked = false;
while (System.currentTimeMillis() - current waitTime) {
svn commit: r1344360 - in /accumulo/branches/1.4/src: ./ core/ core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java server/ server/src/main/java/org/apache/accumulo/server/master/
Author: kturner
Date: Wed May 30 17:39:55 2012
New Revision: 1344360
URL: http://svn.apache.org/viewvc?rev=1344360view=rev
Log:
ACCUMULO-601 made the master advertise an IP addr in zookeeper also made
clients bail when unknownhostexcpetion is thrown when talking to master (merged
from trunk)
Modified:
accumulo/branches/1.4/src/ (props changed)
accumulo/branches/1.4/src/core/ (props changed)
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java
accumulo/branches/1.4/src/server/ (props changed)
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
Propchange: accumulo/branches/1.4/src/
--
Merged /accumulo/trunk:r1344302,1344358
Propchange: accumulo/branches/1.4/src/core/
--
Merged /accumulo/trunk/core:r1344302,1344358
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java?rev=1344360r1=1344359r2=1344360view=diff
==
---
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java
(original)
+++
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java
Wed May 30 17:39:55 2012
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.core.client.impl;
+import java.net.UnknownHostException;
import java.util.List;
import org.apache.accumulo.core.client.AccumuloException;
@@ -34,7 +35,7 @@ import org.apache.thrift.transport.TTran
public class MasterClient {
private static final Logger log = Logger.getLogger(MasterClient.class);
- public static MasterClientService.Iface getConnectionWithRetry(Instance
instance) throws TTransportException {
+ public static MasterClientService.Iface getConnectionWithRetry(Instance
instance) {
ArgumentChecker.notNull(instance);
while (true) {
@@ -64,6 +65,10 @@ public class MasterClient {
instance.getConfiguration());
return client;
} catch (TTransportException tte) {
+ if (tte.getCause().getClass().equals(UnknownHostException.class)) {
+// do not expect to recover from this
+throw new RuntimeException(tte);
+ }
log.debug(Failed to connect to master= + master + portHint= +
portHint + , will retry... , tte);
return null;
}
Propchange: accumulo/branches/1.4/src/server/
--
Merged /accumulo/trunk/server:r1344302,1344358
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1344360r1=1344359r2=1344360view=diff
==
---
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
(original)
+++
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
Wed May 30 17:39:55 2012
@@ -2125,7 +2125,8 @@ public class Master implements LiveTServ
};
long current = System.currentTimeMillis();
final long waitTime =
ServerConfiguration.getSystemConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
-final String masterClientAddress = hostname + : +
ServerConfiguration.getSystemConfiguration().getPort(Property.MASTER_CLIENTPORT);
+final String masterClientAddress =
org.apache.accumulo.core.util.AddressUtil.toString(new
InetSocketAddress(hostname, ServerConfiguration
+.getSystemConfiguration().getPort(Property.MASTER_CLIENTPORT)));
boolean locked = false;
while (System.currentTimeMillis() - current waitTime) {
svn commit: r1345399 - in /accumulo/trunk/core/src/main/java/org/apache/accumulo/core: client/impl/MetadataLocationObtainer.java client/impl/TabletServerBatchReaderIterator.java conf/Property.java
Author: kturner
Date: Sat Jun 2 00:05:13 2012
New Revision: 1345399
URL: http://svn.apache.org/viewvc?rev=1345399view=rev
Log:
ACCUMULO-580 changed how batch scanner buffers data from background threads
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java?rev=1345399r1=1345398r2=1345399view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
Sat Jun 2 00:05:13 2012
@@ -132,8 +132,10 @@ public class MetadataLocationObtainer im
ResultReceiver rr = new ResultReceiver() {
@Override
- public void receive(Key key, Value value) {
-results.put(key, value);
+ public void receive(ListEntryKey,Value entries) {
+for (EntryKey,Value entry : entries) {
+ results.put(entry.getKey(), entry.getValue());
+}
}
};
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java?rev=1345399r1=1345398r2=1345399view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
Sat Jun 2 00:05:13 2012
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.NoSuchElementException;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Semaphore;
@@ -82,8 +83,10 @@ public class TabletServerBatchReaderIter
private final ExecutorService queryThreadPool;
private final ScannerOptions options;
- private ArrayBlockingQueueEntryKey,Value resultsQueue = new
ArrayBlockingQueueEntryKey,Value(1000);
- private EntryKey,Value nextEntry = null;
+ private ArrayBlockingQueueListEntryKey,Value resultsQueue;
+ private IteratorEntryKey,Value batchIterator;
+ private ListEntryKey,Value batch;
+ private static final ListEntryKey,Value LAST_BATCH = new
ArrayListMap.EntryKey,Value();
private Object nextLock = new Object();
private long failSleepTime = 100;
@@ -91,7 +94,7 @@ public class TabletServerBatchReaderIter
private volatile Throwable fatalException = null;
public interface ResultReceiver {
-void receive(Key key, Value value);
+void receive(ListEntryKey,Value entries);
}
private static class MyEntry implements EntryKey,Value {
@@ -131,6 +134,7 @@ public class TabletServerBatchReaderIter
this.numThreads = numThreads;
this.queryThreadPool = queryThreadPool;
this.options = new ScannerOptions(scannerOptions);
+resultsQueue = new ArrayBlockingQueueListEntryKey,Value(numThreads);
if (options.fetchedColumns.size() 0) {
ArrayListRange ranges2 = new ArrayListRange(ranges.size());
@@ -144,14 +148,14 @@ public class TabletServerBatchReaderIter
ResultReceiver rr = new ResultReceiver() {
@Override
- public void receive(Key key, Value value) {
+ public void receive(ListEntryKey,Value entries) {
try {
- resultsQueue.put(new MyEntry(key, value));
+ resultsQueue.put(entries);
} catch (InterruptedException e) {
if
(TabletServerBatchReaderIterator.this.queryThreadPool.isShutdown())
-log.debug(Failed to add Batch Scan result for key + key, e);
+log.debug(Failed to add Batch Scan result, e);
else
-log.warn(Failed to add Batch Scan result for key + key, e);
+log.warn(Failed to add Batch Scan result, e);
fatalException = e;
throw new RuntimeException(e);
@@ -169,17 +173,21 @@ public class TabletServerBatchReaderIter
}
}
+
@Override
public boolean hasNext() {
synchronized (nextLock) {
- // check if one was cached
- if (nextEntry != null)
-return nextEntry.getKey() != null nextEntry.getValue() != null;
+ if (batch
svn commit: r1349503 - in /accumulo/trunk/server/src/main/java/org/apache/accumulo/server/metanalysis: ./ FilterMeta.java FindTablet.java IndexMeta.java LogFileInputFormat.java LogFileOutputFormat.jav
Author: kturner
Date: Tue Jun 12 20:21:11 2012
New Revision: 1349503
URL: http://svn.apache.org/viewvc?rev=1349503view=rev
Log:
ACCUMULO-549 initial checkin of a set of utilities for indexing and analyzing
metadata table mutations found in write ahead logs
Added:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/metanalysis/
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/metanalysis/FilterMeta.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/metanalysis/FindTablet.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/metanalysis/IndexMeta.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/metanalysis/LogFileInputFormat.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/metanalysis/package-info.java
Added:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/metanalysis/FilterMeta.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/metanalysis/FilterMeta.java?rev=1349503view=auto
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/metanalysis/FilterMeta.java
(added)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/metanalysis/FilterMeta.java
Tue Jun 12 20:21:11 2012
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metanalysis;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.server.logger.LogEvents;
+import org.apache.accumulo.server.logger.LogFileKey;
+import org.apache.accumulo.server.logger.LogFileValue;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * A map reduce job that takes a set of walogs and filters out all non
metadata table events.
+ */
+public class FilterMeta extends Configured implements Tool {
+
+ public static class FilterMapper extends
MapperLogFileKey,LogFileValue,LogFileKey,LogFileValue {
+private SetInteger tabletIds;
+
+@Override
+protected void setup(Context context) throws java.io.IOException,
java.lang.InterruptedException {
+ tabletIds = new HashSetInteger();
+}
+
+@Override
+public void map(LogFileKey key, LogFileValue value, Context context)
throws IOException, InterruptedException {
+ if (key.event == LogEvents.OPEN) {
+context.write(key, value);
+ } else if (key.event == LogEvents.DEFINE_TABLET
key.tablet.getTableId().toString().equals(Constants.METADATA_TABLE_ID)) {
+tabletIds.add(key.tid);
+context.write(key, value);
+ } else if ((key.event == LogEvents.MUTATION || key.event ==
LogEvents.MANY_MUTATIONS) tabletIds.contains(key.tid)) {
+context.write(key, value);
+ }
+}
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+
+String jobName = this.getClass().getSimpleName() + _ +
System.currentTimeMillis();
+
+Job job = new Job(getConf(), jobName);
+job.setJarByClass(this.getClass());
+
+Path paths[] = new Path[args.length - 1];
+for (int i = 0; i paths.length; i++) {
+ paths[i] = new Path(args[i]);
+}
+
+job.setInputFormatClass(LogFileInputFormat.class);
+LogFileInputFormat.setInputPaths(job, paths);
+
+job.setOutputFormatClass(LogFileOutputFormat.class);
+LogFileOutputFormat.setOutputPath(job, new Path(args[args.length - 1]));
+
+job.setMapperClass(FilterMapper.class);
+
+job.setNumReduceTasks(0);
+
+job.waitForCompletion(true);
+return job.isSuccessful() ? 0 : 1;
+ }
+
+ public static void main(String
svn commit: r1350669 - /accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
Author: kturner
Date: Fri Jun 15 16:01:49 2012
New Revision: 1350669
URL: http://svn.apache.org/viewvc?rev=1350669view=rev
Log:
ACCUMULO-635 create recovery node in zookeeper during init
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Initialize.java?rev=1350669r1=1350668r2=1350669view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
Fri Jun 15 16:01:49 2012
@@ -363,6 +363,7 @@ public class Initialize {
zoo.putPersistentData(zkInstanceRoot + Constants.ZTABLE_LOCKS, new
byte[0], NodeExistsPolicy.FAIL);
zoo.putPersistentData(zkInstanceRoot + Constants.ZHDFS_RESERVATIONS, new
byte[0], NodeExistsPolicy.FAIL);
zoo.putPersistentData(zkInstanceRoot + Constants.ZNEXT_FILE, new byte[]
{'0'}, NodeExistsPolicy.FAIL);
+zoo.putPersistentData(zkInstanceRoot + Constants.ZRECOVERY, new byte[]
{'0'}, NodeExistsPolicy.FAIL);
}
private static String getInstanceNamePath() throws IOException,
KeeperException, InterruptedException {
svn commit: r1350769 - /accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
Author: kturner
Date: Fri Jun 15 20:08:43 2012
New Revision: 1350769
URL: http://svn.apache.org/viewvc?rev=1350769view=rev
Log:
ACCUMULO-639 tablet server looks for walogs in correct place and does not die
when dir is missing
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1350769r1=1350768r2=1350769view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
Fri Jun 15 20:08:43 2012
@@ -20,6 +20,7 @@ import static org.apache.accumulo.server
import java.io.EOFException;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.management.GarbageCollectorMXBean;
import java.lang.management.ManagementFactory;
@@ -3138,6 +3139,20 @@ public class TabletServer extends Abstra
AccumuloConfiguration conf = serverConf.getConfiguration();
String localWalDirectories = conf.get(Property.LOGGER_DIR);
for (String localWalDirectory : localWalDirectories.split(,)) {
+ if (!localWalDirectory.startsWith(/)) {
+localWalDirectory = System.getenv(ACCUMULO_HOME) + / +
localWalDirectory;
+ }
+
+ FileStatus status = null;
+ try {
+status = localfs.getFileStatus(new Path(localWalDirectory));
+ } catch (FileNotFoundException fne) {}
+
+ if (status == null || !status.isDir()) {
+log.debug(Local walog dir + localWalDirectory + not found );
+continue;
+ }
+
for (FileStatus file : localfs.listStatus(new Path(localWalDirectory))) {
String name = file.getPath().getName();
try {
svn commit: r1350779 - in /accumulo/trunk: server/src/main/java/org/apache/accumulo/server/test/continuous/ test/system/continuous/
Author: kturner
Date: Fri Jun 15 20:19:31 2012
New Revision: 1350779
URL: http://svn.apache.org/viewvc?rev=1350779view=rev
Log:
ACCUMULO-637 a few continuous ingest changes: made # entries written
configurable, made table creation automatic, made log dir creation automatic
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
accumulo/trunk/test/system/continuous/continuous-env.sh.example
accumulo/trunk/test/system/continuous/start-batchwalkers.sh
accumulo/trunk/test/system/continuous/start-ingest.sh
accumulo/trunk/test/system/continuous/start-scanners.sh
accumulo/trunk/test/system/continuous/start-stats.sh
accumulo/trunk/test/system/continuous/start-walkers.sh
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java?rev=1350779r1=1350778r2=1350779view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
Fri Jun 15 20:19:31 2012
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.client.B
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
@@ -67,11 +68,11 @@ public class ContinuousIngest {
args = processOptions(args);
-if (args.length != 13) {
+if (args.length != 14) {
throw new IllegalArgumentException(
usage :
+ ContinuousIngest.class.getName()
- + [--debug debug log] instance name zookeepers user
pass table min max max colf max colq max mem max latency max
threads enable checksum);
+ + [--debug debug log] instance name zookeepers user
pass table num min max max colf max colq max mem max latency
max threads enable checksum);
}
if (debugLog != null) {
@@ -89,16 +90,17 @@ public class ContinuousIngest {
String table = args[4];
-long min = Long.parseLong(args[5]);
-long max = Long.parseLong(args[6]);
-short maxColF = Short.parseShort(args[7]);
-short maxColQ = Short.parseShort(args[8]);
-
-long maxMemory = Long.parseLong(args[9]);
-long maxLatency = Integer.parseInt(args[10]);
-int maxWriteThreads = Integer.parseInt(args[11]);
+long num = Long.parseLong(args[5]);
+long min = Long.parseLong(args[6]);
+long max = Long.parseLong(args[7]);
+short maxColF = Short.parseShort(args[8]);
+short maxColQ = Short.parseShort(args[9]);
+
+long maxMemory = Long.parseLong(args[10]);
+long maxLatency = Integer.parseInt(args[11]);
+int maxWriteThreads = Integer.parseInt(args[12]);
-boolean checksum = Boolean.parseBoolean(args[12]);
+boolean checksum = Boolean.parseBoolean(args[13]);
if (min 0 || max 0 || max = min) {
throw new IllegalArgumentException(bad min and max);
@@ -109,6 +111,11 @@ public class ContinuousIngest {
String path = ZooUtil.getRoot(instance) + Constants.ZTRACERS;
Tracer.getInstance().addReceiver(new ZooSpanClient(zooKeepers, path,
localhost, cingest, 1000));
+if (!conn.tableOperations().exists(table))
+ try {
+conn.tableOperations().create(table);
+ } catch (TableExistsException tee) {}
+
BatchWriter bw = conn.createBatchWriter(table, maxMemory, maxLatency,
maxWriteThreads);
bw = Trace.wrapAll(bw, new CountSampler(1024));
@@ -133,7 +140,7 @@ public class ContinuousIngest {
long lastFlushTime = System.currentTimeMillis();
-while (true) {
+out: while (true) {
// generate first set of nodes
for (int index = 0; index flushInterval; index++) {
long rowLong = genLong(min, max, r);
@@ -152,6 +159,8 @@ public class ContinuousIngest {
}
lastFlushTime = flush(bw, count, flushInterval, lastFlushTime);
+ if (count = num)
+break out;
// generate subsequent sets of nodes that link to previous set of nodes
for (int depth = 1; depth maxDepth; depth++) {
@@ -165,6 +174,8 @@ public class ContinuousIngest {
}
lastFlushTime = flush(bw, count, flushInterval, lastFlushTime);
+if (count = num)
+ break out;
}
// create one big linked list, this makes all of the first inserts
@@ -175,6 +186,8 @@ public class ContinuousIngest
svn commit: r1351425 - in /accumulo/branches/1.4: ./ src/ src/core/ src/server/ src/server/src/main/java/org/apache/accumulo/server/test/continuous/ test/system/continuous/
Author: kturner
Date: Mon Jun 18 17:52:06 2012
New Revision: 1351425
URL: http://svn.apache.org/viewvc?rev=1351425view=rev
Log:
ACCUMULO-637 a few continuous ingest changes: made # entries written
configurable, made table creation automatic, made log dir creation automatic
(merged from trunk)
Modified:
accumulo/branches/1.4/ (props changed)
accumulo/branches/1.4/src/ (props changed)
accumulo/branches/1.4/src/core/ (props changed)
accumulo/branches/1.4/src/server/ (props changed)
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
accumulo/branches/1.4/test/system/continuous/continuous-env.sh.example
accumulo/branches/1.4/test/system/continuous/start-batchwalkers.sh
accumulo/branches/1.4/test/system/continuous/start-ingest.sh
accumulo/branches/1.4/test/system/continuous/start-scanners.sh
accumulo/branches/1.4/test/system/continuous/start-stats.sh
accumulo/branches/1.4/test/system/continuous/start-walkers.sh
Propchange: accumulo/branches/1.4/
--
Merged /accumulo/trunk:r1350779
Propchange: accumulo/branches/1.4/src/
--
Merged /accumulo/trunk/src:r1350779
Merged /accumulo/trunk:r1350779
Propchange: accumulo/branches/1.4/src/core/
--
Merged /accumulo/trunk/src/core:r1350779
Merged /accumulo/trunk/core:r1350779
Propchange: accumulo/branches/1.4/src/server/
--
Merged /accumulo/trunk/src/server:r1350779
Merged /accumulo/trunk/server:r1350779
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java?rev=1351425r1=1351424r2=1351425view=diff
==
---
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
(original)
+++
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
Mon Jun 18 17:52:06 2012
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.client.B
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
@@ -67,11 +68,11 @@ public class ContinuousIngest {
args = processOptions(args);
-if (args.length != 13) {
+if (args.length != 14) {
throw new IllegalArgumentException(
usage :
+ ContinuousIngest.class.getName()
- + [--debug debug log] instance name zookeepers user
pass table min max max colf max colq max mem max latency max
threads enable checksum);
+ + [--debug debug log] instance name zookeepers user
pass table num min max max colf max colq max mem max latency
max threads enable checksum);
}
if (debugLog != null) {
@@ -89,16 +90,17 @@ public class ContinuousIngest {
String table = args[4];
-long min = Long.parseLong(args[5]);
-long max = Long.parseLong(args[6]);
-short maxColF = Short.parseShort(args[7]);
-short maxColQ = Short.parseShort(args[8]);
-
-long maxMemory = Long.parseLong(args[9]);
-long maxLatency = Integer.parseInt(args[10]);
-int maxWriteThreads = Integer.parseInt(args[11]);
+long num = Long.parseLong(args[5]);
+long min = Long.parseLong(args[6]);
+long max = Long.parseLong(args[7]);
+short maxColF = Short.parseShort(args[8]);
+short maxColQ = Short.parseShort(args[9]);
+
+long maxMemory = Long.parseLong(args[10]);
+long maxLatency = Integer.parseInt(args[11]);
+int maxWriteThreads = Integer.parseInt(args[12]);
-boolean checksum = Boolean.parseBoolean(args[12]);
+boolean checksum = Boolean.parseBoolean(args[13]);
if (min 0 || max 0 || max = min) {
throw new IllegalArgumentException(bad min and max);
@@ -109,6 +111,11 @@ public class ContinuousIngest {
String path = ZooUtil.getRoot(instance) + Constants.ZTRACERS;
Tracer.getInstance().addReceiver(new ZooSpanClient(zooKeepers, path,
localhost, cingest, 1000));
+if (!conn.tableOperations().exists(table))
+ try {
+conn.tableOperations().create(table);
+ } catch (TableExistsException tee) {}
+
BatchWriter bw = conn.createBatchWriter
svn commit: r1351452 - in /accumulo/branches/1.4.1: ./ src/ src/core/ src/server/ src/server/src/main/java/org/apache/accumulo/server/test/continuous/ test/system/continuous/
Author: kturner Date: Mon Jun 18 18:29:00 2012 New Revision: 1351452 URL: http://svn.apache.org/viewvc?rev=1351452view=rev Log: ACCUMULO-637 ACCUMULO-614 merge changes from 1.4 branch Modified: accumulo/branches/1.4.1/ (props changed) accumulo/branches/1.4.1/CHANGES accumulo/branches/1.4.1/src/ (props changed) accumulo/branches/1.4.1/src/core/ (props changed) accumulo/branches/1.4.1/src/server/ (props changed) accumulo/branches/1.4.1/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java accumulo/branches/1.4.1/test/system/continuous/continuous-env.sh.example accumulo/branches/1.4.1/test/system/continuous/start-batchwalkers.sh accumulo/branches/1.4.1/test/system/continuous/start-ingest.sh accumulo/branches/1.4.1/test/system/continuous/start-scanners.sh accumulo/branches/1.4.1/test/system/continuous/start-stats.sh accumulo/branches/1.4.1/test/system/continuous/start-walkers.sh Propchange: accumulo/branches/1.4.1/ -- Merged /accumulo/trunk:r1350779 Merged /accumulo/branches/1.4:r1346082-1346088,1346090-1346833,1346835-1349971,1349973-1351449 Modified: accumulo/branches/1.4.1/CHANGES URL: http://svn.apache.org/viewvc/accumulo/branches/1.4.1/CHANGES?rev=1351452r1=1351451r2=1351452view=diff == --- accumulo/branches/1.4.1/CHANGES (original) +++ accumulo/branches/1.4.1/CHANGES Mon Jun 18 18:29:00 2012 @@ -1,3 +1,62 @@ +Release Notes - Apache Accumulo - Version 1.4.1 + +** Bug +* [ACCUMULO-355] - functional tests should run under a non-hadoop user +* [ACCUMULO-398] - Table tablets not evenly spread. +* [ACCUMULO-428] - loggers are not closed when high hold times are detected +* [ACCUMULO-449] - Failed log copy is not restarted +* [ACCUMULO-468] - low-memory warning message is incorrect +* [ACCUMULO-487] - Batch Scanner can get stuck when external thread closes scanner +* [ACCUMULO-488] - InputFormats' RecordReaders should call Context.progress +* [ACCUMULO-496] - Cloudtrace.thrift needs to have it's namespace updated to the new package +* [ACCUMULO-499] - Garbage collector should call Tracer.offNoFlush instead of Tracer.off +* [ACCUMULO-505] - % of Used DFS is overflowing on a large cluster +* [ACCUMULO-509] - default walog copy/sort uses replication of 1 +* [ACCUMULO-512] - DEFAULT_MAX_LATENCY in AccumuloOutputFormat wrong units +* [ACCUMULO-516] - Column family search with sparse files is painfully long +* [ACCUMULO-520] - assigned tablets are not considered offline +* [ACCUMULO-521] - examples functional test is using the wrong class name +* [ACCUMULO-533] - make system iterators thread-safer +* [ACCUMULO-542] - with a large root tablet, tablet servers will get stuck loading other metadata tablets +* [ACCUMULO-561] - Monitor scan mb/s does not show batch scan activity +* [ACCUMULO-562] - Multi term grep in shell fails +* [ACCUMULO-566] - monitor should display zero tablet servers as red, even if there's only one tablet server configured +* [ACCUMULO-570] - dirlist example README is a little inconsistent +* [ACCUMULO-585] - Complain loudly when env file is missing +* [ACCUMULO-591] - Need to clear tablet location cache before computing input splits in input format +* [ACCUMULO-592] - We lost the security policy example +* [ACCUMULO-601] - client fails to resolve master hostname when not using fully qualified domain name +* [ACCUMULO-615] - accumulo runs fine on Hadoop 1.0 +* [ACCUMULO-633] - FirstEntryInRowIterator is broken and has no test + +** Improvement +* [ACCUMULO-337] - Make logging an example file +* [ACCUMULO-364] - Security Policy file should be an example +* [ACCUMULO-399] - need a randomwalk node that checks tablet balance +* [ACCUMULO-459] - Accumulo init should fail fast +* [ACCUMULO-471] - allow wikisearch ingest to run on uncompressed input +* [ACCUMULO-480] - config.html should be more clear with regards to needs restart +* [ACCUMULO-489] - Input Format puts Base64 encoded passwords in Configuration, which is world readable +* [ACCUMULO-494] - Document relationship between max walogs and max tserver memory +* [ACCUMULO-502] - Document what replaces the deprecated PerColumnIteratorConfig class. +* [ACCUMULO-503] - start-here does unecessary SSHs +* [ACCUMULO-510] - Rename scanner threads to identify table being scanned +* [ACCUMULO-550] - Colocate rfile index entries within file +* [ACCUMULO-554] - monitor should display red when time-since last report is higher than a few minutes +* [ACCUMULO-573] - Fix grammar in shell commands +* [ACCUMULO-574] - Document reseek of iterators +* [ACCUMULO-637] - Make entries written configurable for continuous ingest + +** New
svn commit: r1351844 - in /accumulo/trunk/server/src/main/java/org/apache/accumulo/server: master/Master.java tabletserver/log/LogSorter.java
Author: kturner
Date: Tue Jun 19 20:41:03 2012
New Revision: 1351844
URL: http://svn.apache.org/viewvc?rev=1351844view=rev
Log:
ACCUMULO-649 fixed a few log recovery issues
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1351844r1=1351843r2=1351844view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
Tue Jun 19 20:41:03 2012
@@ -2072,6 +2072,12 @@ public class Master implements LiveTServ
@Override
public void process(WatchedEvent event) {
nextEvent.event(Noticed recovery changes, event.getType());
+try {
+ // watcher only fires once, add it back
+ ZooReaderWriter.getInstance().getChildren(zroot +
Constants.ZRECOVERY, this);
+} catch (Exception e) {
+ log.error(Failed to add log recovery watcher back, e);
+}
}
});
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java?rev=1351844r1=1351843r2=1351844view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
Tue Jun 19 20:41:03 2012
@@ -113,7 +113,7 @@ public class LogSorter {
}
}
fs.create(new Path(destPath, finished)).close();
-log.info(Log copy/sort of + name + complete);
+log.debug(Log copy/sort of + name + complete);
} catch (Throwable t) {
try {
fs.create(new Path(destPath, failed)).close();
@@ -201,7 +201,7 @@ public class LogSorter {
case NodeChildrenChanged:
if (event.getPath().equals(path))
try {
-attemptRecoveries(zoo, serverName, path,
zoo.getChildren(path));
+attemptRecoveries(zoo, serverName, path, zoo.getChildren(path,
this));
} catch (KeeperException e) {
log.error(Unable to get recovery information, e);
} catch (InterruptedException e) {
@@ -237,10 +237,14 @@ public class LogSorter {
}, r.nextInt(1000), 60 * 1000);
}
- private void attemptRecoveries(final ZooReaderWriter zoo, final String
serverName, String path, ListString children) {
+ private void attemptRecoveries(final ZooReaderWriter zoo, final String
serverName, final String path, ListString children) {
if (children.size() == 0)
return;
-log.info(Zookeeper references + children.size() + recoveries,
attempting locks);
+
+if (threadPool.getQueue().size() 1)
+ return;
+
+log.debug(Zookeeper references + children.size() + recoveries,
attempting locks);
Random random = new Random();
Collections.shuffle(children, random);
try {
@@ -258,9 +262,9 @@ public class LogSorter {
if (threadPool.getQueue().size() 1) {
lock.unlock();
log.debug(got the lock, but thread pool is busy; released the
lock on + child);
-continue;
+break;
}
- log.info(got lock for + child);
+ log.debug(got lock for + child);
byte[] contents = zoo.getData(childPath, null);
String destination = Constants.getRecoveryDir(conf) + / + child;
startSort(new String(contents), destination, new LogSortNotifier() {
@@ -272,10 +276,17 @@ public class LogSorter {
} catch (Exception e) {
log.error(Error received when trying to delete recovery entry
in zookeeper + childPath);
}
+ try {
+attemptRecoveries(zoo, serverName, path,
zoo.getChildren(path));
+ } catch (KeeperException e) {
+log.error(Unable to get recovery information, e);
+ } catch (InterruptedException e) {
+log.info(Interrupted getting recovery information, e);
+ }
}
});
} else {
- log.info(failed to get the lock + child);
+ log.debug(failed to get the lock + child);
}
}
} catch (Throwable t) {
svn commit: r1351885 - /accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
Author: kturner
Date: Tue Jun 19 22:19:21 2012
New Revision: 1351885
URL: http://svn.apache.org/viewvc?rev=1351885view=rev
Log:
ACCUMULO-628 fixed unintended deletion of tablets last location
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java?rev=1351885r1=1351884r2=1351885view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
Tue Jun 19 22:19:21 2012
@@ -191,12 +191,12 @@ public class MetadataTable extends org.a
if (dfv.getNumEntries() 0) {
m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(path), new
Value(dfv.encode()));
ColumnFQ.put(m, Constants.METADATA_TIME_COLUMN, new
Value(time.getBytes()));
- // erase the old location
- if (lastLocation != null)
-lastLocation.clearLastLocation(m);
// stuff in this location
TServerInstance self = getTServerInstance(address, zooLock);
self.putLastLocation(m);
+ // erase the old location
+ if (lastLocation != null !lastLocation.equals(self))
+lastLocation.clearLastLocation(m);
}
if (unusedWalLogs != null) {
for (String entry : unusedWalLogs) {
@@ -465,11 +465,12 @@ public class MetadataTable extends org.a
if (compactionId != null)
ColumnFQ.put(m, Constants.METADATA_COMPACT_COLUMN, new Value(( +
compactionId).getBytes()));
-// remove the old location
-if (lastLocation != null)
- lastLocation.clearLastLocation(m);
TServerInstance self = getTServerInstance(address, zooLock);
self.putLastLocation(m);
+
+// remove the old location
+if (lastLocation != null !lastLocation.equals(self))
+ lastLocation.clearLastLocation(m);
update(credentials, zooLock, m);
}
svn commit: r1356891 - in /accumulo/branches/1.4: src/core/src/main/java/org/apache/accumulo/core/ src/core/src/main/java/org/apache/accumulo/core/conf/ src/core/src/main/java/org/apache/accumulo/core
Author: kturner
Date: Tue Jul 3 19:46:50 2012
New Revision: 1356891
URL: http://svn.apache.org/viewvc?rev=1356891view=rev
Log:
ACCUMULO-409 Make tservers copy failed bulk import files instead of master.
Added:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/BulkFailedCopyProcessor.java
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/Constants.java
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooUtil.java
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/zookeeper/IZooReaderWriter.java
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
accumulo/branches/1.4/test/system/test4/bulk_import_test.sh
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/Constants.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/Constants.java?rev=1356891r1=1356890r2=1356891view=diff
==
---
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/Constants.java
(original)
+++
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/Constants.java
Tue Jul 3 19:46:50 2012
@@ -77,6 +77,8 @@ public class Constants {
public static final String ZNEXT_FILE = /next_file;
+ public static final String ZBULK_FAILED_COPYQ = /bulk_failed_copyq;
+
public static final String ZHDFS_RESERVATIONS = /hdfs_reservations;
public static final String METADATA_TABLE_ID = !0;
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1356891r1=1356890r2=1356891view=diff
==
---
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
(original)
+++
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
Tue Jul 3 19:46:50 2012
@@ -153,7 +153,9 @@ public enum Property {
TSERV_HOLD_TIME_SUICIDE(tserver.hold.time.max, 5m,
PropertyType.TIMEDURATION,
The maximum time for a tablet server to be in the \memory full\
state. If the tablet server cannot write out memory
+ in this much time, it will assume there is some failure local to
its node, and quit. A value of zero is equivalent to forever.),
-
+ TSERV_WORKQ_THREADS(tserver.workq.threads, 2, PropertyType.COUNT,
+ The number of threads for the distributed workq. These threads are
used for copying failed bulk files.),
+
// properties that are specific to logger server behavior
LOGGER_PREFIX(logger., null, PropertyType.PREFIX, Properties in this
category affect the behavior of the write-ahead logger servers),
LOGGER_PORT(logger.port.client, 11224, PropertyType.PORT, The port used
for write-ahead logger services),
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooUtil.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooUtil.java?rev=1356891r1=1356890r2=1356891view=diff
==
---
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooUtil.java
(original)
+++
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooUtil.java
Tue Jul 3 19:46:50 2012
@@ -221,6 +221,10 @@ public class ZooUtil {
return zk.create(zPath, data, ZooUtil.PUBLIC,
CreateMode.PERSISTENT_SEQUENTIAL);
}
+ public static String putEphemeralData(ZooKeeper zk, String zPath, byte[]
data) throws KeeperException, InterruptedException {
+return zk.create(zPath, data, ZooUtil.PUBLIC, CreateMode.EPHEMERAL);
+ }
+
public static String putEphemeralSequential(ZooKeeper zk, String zPath,
byte[] data) throws KeeperException, InterruptedException {
return zk.create(zPath, data, ZooUtil.PUBLIC,
CreateMode.EPHEMERAL_SEQUENTIAL);
}
@@ -258,5 +262,4 @@ public class ZooUtil {
Stat stat = zk.exists(lid.path + / + lid.node, false);
return stat != null stat.getEphemeralOwner
svn commit: r1356949 - in /accumulo/trunk: ./ core/ core/src/main/java/org/apache/accumulo/core/ core/src/main/java/org/apache/accumulo/core/conf/ core/src/test/java/org/apache/accumulo/core/iterators
Author: kturner
Date: Tue Jul 3 21:13:15 2012
New Revision: 1356949
URL: http://svn.apache.org/viewvc?rev=1356949view=rev
Log:
ACCUMULO-409 Make tservers copy failed bulk import files instead of master.
(merged from 1.4)
Added:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/BulkFailedCopyProcessor.java
- copied unchanged from r1356900,
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/BulkFailedCopyProcessor.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
- copied, changed from r1356900,
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
Modified:
accumulo/trunk/ (props changed)
accumulo/trunk/core/ (props changed)
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/FirstEntryInRowTest.java
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/IZooReaderWriter.java
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReaderWriter.java
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooUtil.java
accumulo/trunk/server/ (props changed)
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
accumulo/trunk/src/ (props changed)
accumulo/trunk/test/system/test4/bulk_import_test.sh
Propchange: accumulo/trunk/
--
Merged /accumulo/trunk/src:r1341000,1342373,1351691,1356400
Merged
/accumulo/branches/1.4/src:r1339309-1342420,1343897-1343898,1343943-1349971,1349973-1351424,1351426-1354669,1354673-1356900
Merged /accumulo/branches/1.3:r1354669
Merged /accumulo/branches/1.3/src:r1354669
Merged
/accumulo/branches/1.4:r1343943-1349971,1349973-1351424,1351426-1354668,1354673-1356923
Propchange: accumulo/trunk/core/
--
Merged /accumulo/trunk/src/core:r1341000,1342373,1351691,1356400
Merged /accumulo/branches/1.3/src/core:r1354669
Merged
/accumulo/branches/1.4/core:r1343943-1349971,1349973-1351424,1351426-1354668,1354673-1356923
Merged
/accumulo/branches/1.4/src/core:r1339309-1342420,1343897-1343898,1343943-1349971,1349973-1351424,1351426-1354669,1354673-1356900
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java?rev=1356949r1=1356948r2=1356949view=diff
==
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java
Tue Jul 3 21:13:15 2012
@@ -74,6 +74,8 @@ public class Constants {
public static final String ZNEXT_FILE = /next_file;
+ public static final String ZBULK_FAILED_COPYQ = /bulk_failed_copyq;
+
public static final String ZHDFS_RESERVATIONS = /hdfs_reservations;
public static final String ZRECOVERY = /recovery;
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1356949r1=1356948r2=1356949view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
Tue Jul 3 21:13:15 2012
@@ -151,7 +151,9 @@ public enum Property {
TSERV_RECOVERY_MAX_CONCURRENT(tserver.recovery.concurrent.max, 2,
PropertyType.COUNT, The maximum number of threads to use to sort logs during
recovery),
TSERV_SORT_BUFFER_SIZE(tserver.sort.buffer.size, 200M,
PropertyType.MEMORY, The amount of memory to use when sorting logs during
recovery.),
TSERV_ARCHIVE_WALOGS(tserver.archive.walogs, false,
PropertyType.BOOLEAN, Keep copies of the WALOGs for debugging purposes),
-
+ TSERV_WORKQ_THREADS(tserver.workq.threads, 2, PropertyType.COUNT,
+ The number of threads for the distributed workq. These threads are
used for copying failed bulk files.),
+
// properties that are specific to logger server behavior
LOGGER_PREFIX(logger., null, PropertyType.PREFIX, Properties in this
category affect the behavior of the write-ahead logger servers
svn commit: r1357721 - in /accumulo/trunk/server/src/main/java/org/apache/accumulo/server: master/recovery/SubmitFileForRecovery.java tabletserver/TabletServer.java tabletserver/log/LogSorter.java
Author: kturner
Date: Thu Jul 5 16:19:05 2012
New Revision: 1357721
URL: http://svn.apache.org/viewvc?rev=1357721view=rev
Log:
ACCUMULO-409 Modified log recovery to use new DistributedWorkQueue
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/recovery/SubmitFileForRecovery.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/recovery/SubmitFileForRecovery.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/recovery/SubmitFileForRecovery.java?rev=1357721r1=1357720r2=1357721view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/recovery/SubmitFileForRecovery.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/recovery/SubmitFileForRecovery.java
Thu Jul 5 16:19:05 2012
@@ -21,9 +21,9 @@ import java.io.IOException;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.Repo;
-import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.server.master.Master;
import org.apache.accumulo.server.master.tableOps.MasterRepo;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -48,18 +48,19 @@ public class SubmitFileForRecovery exten
public RepoMaster call(long tid, final Master master) throws Exception {
master.updateRecoveryInProgress(file);
String source = RecoverLease.getSource(master, server, file).toString();
+new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) +
Constants.ZRECOVERY).addWork(file, source.getBytes());
+
ZooReaderWriter zoo = ZooReaderWriter.getInstance();
final String path = ZooUtil.getRoot(master.getInstance()) +
Constants.ZRECOVERY + / + file;
-zoo.putPersistentData(path, source.getBytes(), NodeExistsPolicy.SKIP);
log.info(Created zookeeper entry + path + with data + source);
zoo.exists(path, new Watcher() {
@Override
public void process(WatchedEvent event) {
switch (event.getType()) {
- case NodeDataChanged:
+ case NodeDeleted:
log.info(noticed recovery entry for + file + was removed);
FileSystem fs = master.getFileSystem();
-Path finished = new
Path(Constants.getRecoveryDir(master.getSystemConfiguration()), finished);
+Path finished = new
Path(Constants.getRecoveryDir(master.getSystemConfiguration()) + / + file,
finished);
try {
if (fs.exists(finished))
log.info(log recovery for + file + successful);
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1357721r1=1357720r2=1357721view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
Thu Jul 5 16:19:05 2012
@@ -51,7 +51,6 @@ import java.util.concurrent.ArrayBlockin
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.RunnableFuture;
import java.util.concurrent.ThreadPoolExecutor;
@@ -126,9 +125,9 @@ import org.apache.accumulo.core.util.Cac
import org.apache.accumulo.core.util.ColumnFQ;
import org.apache.accumulo.core.util.Daemon;
import org.apache.accumulo.core.util.LoggingRunnable;
-import org.apache.accumulo.core.util.NamingThreadFactory;
import org.apache.accumulo.core.util.ServerServices;
import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.util.SimpleThreadPool;
import org.apache.accumulo.core.util.Stat;
import org.apache.accumulo.core.util.ThriftUtil;
import org.apache.accumulo.core.util.UtilWaitThread;
@@ -2714,22 +2713,22 @@ public class TabletServer extends Abstra
}
clientAddress = new InetSocketAddress(clientAddress.getAddress(),
clientPort);
announceExistence();
-try {
- logSorter.startWatchingForRecoveryLogs(getClientAddressString());
-} catch (Exception ex) {
- log.error(Error setting
svn commit: r1370914 - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/client/ core/src/main/java/org/apache/accumulo/core/client/impl/ core/src/main/java/org/apache/accumulo/core/clie
Author: kturner
Date: Wed Aug 8 19:47:57 2012
New Revision: 1370914
URL: http://svn.apache.org/viewvc?rev=1370914view=rev
Log:
ACCUMULO-705 added timeout to batch scanner
Added:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/TimedOutException.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/TimeoutTest.java
accumulo/trunk/test/system/auto/simple/timeout.py
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchScanner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/Scanner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerImpl.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReader.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchScanner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/ThriftUtil.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchScanner.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchScanner.java?rev=1370914r1=1370913r2=1370914view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchScanner.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchScanner.java
Wed Aug 8 19:47:57 2012
@@ -44,4 +44,17 @@ public interface BatchScanner extends Sc
* Cleans up and finalizes the scanner
*/
void close();
+
+ /**
+ * Sets a timeout threshold for a server to respond. The batch scanner will
accomplish as much work as possible before throwing an exception. BatchScanner
+ * iterators will throw a {@link TimedOutException} when all needed servers
timeout.
+ *
+ * p
+ * If not set, the timeout defaults to MAX_INT
+ *
+ * @param timeout
+ * in seconds
+ */
+ @Override
+ void setTimeOut(int timeout);
}
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/Scanner.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/Scanner.java?rev=1370914r1=1370913r2=1370914view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/Scanner.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/Scanner.java
Wed Aug 8 19:47:57 2012
@@ -28,21 +28,6 @@ import org.apache.accumulo.core.data.Ran
public interface Scanner extends ScannerBase {
/**
- * This setting determines how long a scanner will automatically retry when
a failure occurs. By default a scanner will retry forever.
- *
- * @param timeOut
- * in seconds
- */
- public void setTimeOut(int timeOut);
-
- /**
- * Returns the setting for how long a scanner will automatically retry when
a failure occurs.
- *
- * @return the timeout configured for this scanner
- */
- public int getTimeOut();
-
- /**
* Sets the range of keys to scan over.
*
* @param range
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java?rev=1370914r1=1370913r2=1370914view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
Wed Aug 8 19:47:57 2012
@@ -100,4 +100,19 @@ public interface ScannerBase extends Ite
* @return an iterator over Key,Value pairs which meet the restrictions set
on the scanner
*/
public IteratorEntryKey,Value iterator();
+
+ /**
+ * This setting determines how long a scanner will automatically retry when
a failure occurs. By default a scanner will retry forever.
+ *
+ * @param timeOut
+ * in seconds
+ */
+ public void setTimeOut(int timeOut);
+
+ /**
+ * Returns the setting for how long a scanner will automatically retry when
a failure occurs.
+ *
+ * @return the timeout configured for this scanner
+ */
+ public int getTimeOut();
}
Added:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/TimedOutException.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client
svn commit: r1370925 [2/2] - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/ core/src/main/java/org/apache/accumulo/core/client/admin/ core/src/main/java/org/apache/accumulo/core/clie
Added:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/ImportTable.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/ImportTable.java?rev=1370925view=auto
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/ImportTable.java
(added)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/ImportTable.java
Wed Aug 8 20:02:58 2012
@@ -0,0 +1,595 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.tableOps;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.admin.TableOperationsImpl;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import
org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.fate.Repo;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.master.Master;
+import org.apache.accumulo.server.master.state.tables.TableManager;
+import org.apache.accumulo.server.security.Authenticator;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.security.ZKAuthenticator;
+import org.apache.accumulo.server.tabletserver.UniqueNameAllocator;
+import org.apache.accumulo.server.test.FastFormat;
+import org.apache.accumulo.server.util.MetadataTable;
+import org.apache.accumulo.server.util.TablePropUtil;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+/**
+ *
+ */
+class ImportedTableInfo implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public String exportDir;
+ public String user;
+ public String tableName;
+ public String tableId;
+ public String importDir;
+}
+
+class FinishImportTable extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ public FinishImportTable(ImportedTableInfo ti) {
+this.tableInfo = ti;
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+return 0;
+ }
+
+ @Override
+ public RepoMaster call(long tid, Master env) throws Exception {
+
+env.getFileSystem().delete(new Path(tableInfo.importDir, mappings.txt),
true);
+
+TableManager.getInstance().transitionTableState(tableInfo.tableId,
TableState.ONLINE);
+
+Utils.unreserveTable(tableInfo.tableId, tid, true);
+
+Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(),
tid);
+
+env.getEventCoordinator().event(Imported table %s , tableInfo.tableName);
+
+Logger.getLogger(FinishImportTable.class).debug(Imported table +
tableInfo.tableId + + tableInfo.tableName);
+
+return null;
svn commit: r1376048 - /accumulo/site/trunk/content/get_involved.mdtext
Author: kturner Date: Wed Aug 22 13:53:20 2012 New Revision: 1376048 URL: http://svn.apache.org/viewvc?rev=1376048view=rev Log: CMS commit to accumulo by kturner Modified: accumulo/site/trunk/content/get_involved.mdtext Modified: accumulo/site/trunk/content/get_involved.mdtext URL: http://svn.apache.org/viewvc/accumulo/site/trunk/content/get_involved.mdtext?rev=1376048r1=1376047r2=1376048view=diff == --- accumulo/site/trunk/content/get_involved.mdtext (original) +++ accumulo/site/trunk/content/get_involved.mdtext Wed Aug 22 13:53:20 2012 @@ -44,7 +44,9 @@ as easy as possible for people to get in ## Mailing lists and Meetups Your first engagement with the project should be to subscribe to our -[mailing lists][1]. You can also [look for Accumulo Meetups][7] in your area. +[mailing lists][1]. You can also [look for Accumulo Meetups][7] in your area. +Apache Accumulo developers and community members hang out in the #accumulo +channel on irc.freenode.net. ## Developer's Guide
svn commit: r1381426 [2/2] - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/client/impl/thrift/ core/src/main/java/org/apache/accumulo/core/data/thrift/ core/src/main/java/org/apache/
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TIteratorSetting.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TIteratorSetting.java?rev=1381426r1=1381425r2=1381426view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TIteratorSetting.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TIteratorSetting.java
Wed Sep 5 22:34:40 2012
@@ -47,7 +47,7 @@ import org.slf4j.LoggerFactory;
public MapString,String properties; // required
/** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ @SuppressWarnings(all) public enum _Fields implements
org.apache.thrift.TFieldIdEnum {
PRIORITY((short)1, priority),
NAME((short)2, name),
ITERATOR_CLASS((short)3, iteratorClass),
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java?rev=1381426r1=1381425r2=1381426view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
Wed Sep 5 22:34:40 2012
@@ -2306,7 +2306,7 @@ import org.slf4j.LoggerFactory;
public boolean isolated; // required
/** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
-public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+@SuppressWarnings(all) public enum _Fields implements
org.apache.thrift.TFieldIdEnum {
TINFO((short)11, tinfo),
CREDENTIALS((short)1, credentials),
EXTENT((short)2, extent),
@@ -3916,7 +3916,7 @@ import org.slf4j.LoggerFactory;
public TooManyFilesException tmfe; // required
/** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
-public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+@SuppressWarnings(all) public enum _Fields implements
org.apache.thrift.TFieldIdEnum {
SUCCESS((short)0, success),
SEC((short)1, sec),
NSTE((short)2, nste),
@@ -4573,7 +4573,7 @@ import org.slf4j.LoggerFactory;
public long scanID; // required
/** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
-public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+@SuppressWarnings(all) public enum _Fields implements
org.apache.thrift.TFieldIdEnum {
TINFO((short)2, tinfo),
SCAN_ID((short)1, scanID);
@@ -5031,7 +5031,7 @@ import org.slf4j.LoggerFactory;
public TooManyFilesException tmfe; // required
/** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
-public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+@SuppressWarnings(all) public enum _Fields implements
org.apache.thrift.TFieldIdEnum {
SUCCESS((short)0, success),
NSSI((short)1, nssi),
NSTE((short)2, nste),
@@ -5688,7 +5688,7 @@ import org.slf4j.LoggerFactory;
public long scanID; // required
/** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
-public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+@SuppressWarnings(all) public enum _Fields implements
org.apache.thrift.TFieldIdEnum {
TINFO((short)2, tinfo),
SCAN_ID((short)1, scanID);
@@ -6154,7 +6154,7 @@ import org.slf4j.LoggerFactory;
public boolean waitForWrites; // required
/** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
-public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+@SuppressWarnings(all) public enum _Fields implements
org.apache.thrift.TFieldIdEnum {
TINFO((short)8, tinfo),
CREDENTIALS((short)1, credentials),
BATCH((short)2, batch),
@@ -7559,7 +7559,7 @@ import org.slf4j.LoggerFactory;
public org.apache.accumulo.core.security.thrift.ThriftSecurityException
sec; // required
/** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
-public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+@SuppressWarnings(all) public enum _Fields implements
org.apache.thrift.TFieldIdEnum {
svn commit: r1387182 [3/3] - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/client/ core/src/main/java/org/apache/accumulo/core/client/impl/ core/src/main/java/org/apache/accumulo/cor
Modified: accumulo/trunk/server/src/test/java/org/apache/accumulo/server/util/CloneTest.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/test/java/org/apache/accumulo/server/util/CloneTest.java?rev=1387182r1=1387181r2=1387182view=diff == --- accumulo/trunk/server/src/test/java/org/apache/accumulo/server/util/CloneTest.java (original) +++ accumulo/trunk/server/src/test/java/org/apache/accumulo/server/util/CloneTest.java Tue Sep 18 14:25:37 2012 @@ -23,6 +23,7 @@ import junit.framework.TestCase; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.mock.MockInstance; @@ -44,13 +45,13 @@ public class CloneTest extends TestCase Constants.METADATA_TIME_COLUMN.put(mut, new Value(M0.getBytes())); Constants.METADATA_DIRECTORY_COLUMN.put(mut, new Value(/default_tablet.getBytes())); -BatchWriter bw1 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000, 6l, 1); +BatchWriter bw1 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig()); bw1.addMutation(mut); bw1.close(); -BatchWriter bw2 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000, 6l, 1); +BatchWriter bw2 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig()); MetadataTable.initializeClone(0, 1, conn, bw2); @@ -73,13 +74,13 @@ public class CloneTest extends TestCase Constants.METADATA_DIRECTORY_COLUMN.put(mut, new Value(/default_tablet.getBytes())); mut.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY.toString(), /default_tablet/0_0.rf, 1,200); -BatchWriter bw1 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000, 6l, 1); +BatchWriter bw1 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig()); bw1.addMutation(mut); bw1.flush(); -BatchWriter bw2 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000, 6l, 1); +BatchWriter bw2 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig()); MetadataTable.initializeClone(0, 1, conn, bw2); @@ -118,13 +119,13 @@ public class CloneTest extends TestCase MockInstance mi = new MockInstance(); Connector conn = mi.getConnector(, ); -BatchWriter bw1 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000, 6l, 1); +BatchWriter bw1 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig()); bw1.addMutation(createTablet(0, null, null, /default_tablet, /default_tablet/0_0.rf)); bw1.flush(); -BatchWriter bw2 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000, 6l, 1); +BatchWriter bw2 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig()); MetadataTable.initializeClone(0, 1, conn, bw2); @@ -160,13 +161,13 @@ public class CloneTest extends TestCase MockInstance mi = new MockInstance(); Connector conn = mi.getConnector(, ); -BatchWriter bw1 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000, 6l, 1); +BatchWriter bw1 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig()); bw1.addMutation(createTablet(0, null, null, /default_tablet, /default_tablet/0_0.rf)); bw1.flush(); -BatchWriter bw2 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000, 6l, 1); +BatchWriter bw2 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig()); MetadataTable.initializeClone(0, 1, conn, bw2); @@ -231,14 +232,14 @@ public class CloneTest extends TestCase MockInstance mi = new MockInstance(); Connector conn = mi.getConnector(, ); -BatchWriter bw1 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000, 6l, 1); +BatchWriter bw1 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig()); bw1.addMutation(createTablet(0, m, null, /d1, /d1/file1)); bw1.addMutation(createTablet(0, null, m, /d2, /d2/file2)); bw1.flush(); -BatchWriter bw2 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000, 6l, 1); +BatchWriter bw2 = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig()); MetadataTable.initializeClone(0, 1, conn, bw2); @@ -278,14 +279,14 @@ public class CloneTest extends TestCase MockInstance mi = new MockInstance(); Connector conn = mi.getConnector(, ); -BatchWriter bw1 =
svn commit: r1387249 - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/client/ core/src/main/java/org/apache/accumulo/core/client/impl/ core/src/main/java/org/apache/accumulo/core/clie
Author: kturner
Date: Tue Sep 18 16:48:00 2012
New Revision: 1387249
URL: http://svn.apache.org/viewvc?rev=1387249view=rev
Log:
ACCUMULO-705 Modfied API so that setTimeout() method is consistent across
Scanner, BatchScanner, and BatchWriter
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchScanner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/IsolatedScanner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/Scanner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerImpl.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReader.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchScanner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockScanner.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/trace/NullScanner.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/TimeoutTest.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchScanner.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchScanner.java?rev=1387249r1=1387248r2=1387249view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchScanner.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/BatchScanner.java
Tue Sep 18 16:48:00 2012
@@ -17,6 +17,7 @@
package org.apache.accumulo.core.client;
import java.util.Collection;
+import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.data.Range;
@@ -47,14 +48,16 @@ public interface BatchScanner extends Sc
/**
* Sets a timeout threshold for a server to respond. The batch scanner will
accomplish as much work as possible before throwing an exception. BatchScanner
- * iterators will throw a {@link TimedOutException} when all needed servers
timeout.
+ * iterators will throw a {@link TimedOutException} when all needed servers
timeout. Setting the timeout to zero or Long.MAX_VALUE and TimeUnit.MILLISECONDS
+ * means no timeout.
*
* p
- * If not set, the timeout defaults to MAX_INT
+ * If not set, there is not timeout. The BatchScanner will retry forever.
*
* @param timeout
- * in seconds
+ * @param timeUnit
+ * determines how timeout is interpreted
*/
@Override
- void setTimeOut(int timeout);
+ void setTimeout(long timeout, TimeUnit timeUnit);
}
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java?rev=1387249r1=1387248r2=1387249view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
Tue Sep 18 16:48:00 2012
@@ -24,6 +24,7 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.client.impl.ScannerOptions;
import org.apache.accumulo.core.client.mock.IteratorAdapter;
@@ -53,7 +54,6 @@ import org.apache.hadoop.io.Text;
*/
public class ClientSideIteratorScanner extends ScannerOptions implements
Scanner {
private int size;
- private int timeOut;
private Range range;
private boolean isolated = false;
@@ -136,7 +136,7 @@ public class ClientSideIteratorScanner e
smi = new ScannerTranslator(scanner);
this.range = scanner.getRange();
this.size = scanner.getBatchSize();
-this.timeOut = scanner.getTimeOut();
+this.timeOut = scanner.getTimeout(TimeUnit.MILLISECONDS);
}
/**
@@ -151,7 +151,7 @@ public class ClientSideIteratorScanner e
@Override
public IteratorEntryKey,Value iterator() {
smi.scanner.setBatchSize(size);
-smi.scanner.setTimeOut(timeOut);
+smi.scanner.setTimeout(timeOut, TimeUnit.MILLISECONDS);
if (isolated)
smi.scanner.enableIsolation();
else
@@ -208,13 +208,19 @@ public class ClientSideIteratorScanner e
}
@Override
- public
svn commit: r1388194 - /accumulo/trunk/test/compat/diffAPI.pl
Author: kturner
Date: Thu Sep 20 20:16:07 2012
New Revision: 1388194
URL: http://svn.apache.org/viewvc?rev=1388194view=rev
Log:
ACCUMULO-765 updated diffAPI.pl to show which methods are deprecated
Modified:
accumulo/trunk/test/compat/diffAPI.pl
Modified: accumulo/trunk/test/compat/diffAPI.pl
URL:
http://svn.apache.org/viewvc/accumulo/trunk/test/compat/diffAPI.pl?rev=1388194r1=1388193r2=1388194view=diff
==
--- accumulo/trunk/test/compat/diffAPI.pl (original)
+++ accumulo/trunk/test/compat/diffAPI.pl Thu Sep 20 20:16:07 2012
@@ -15,6 +15,60 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+sub trim($)
+{
+ my $string = shift;
+ $string =~ s/^\s+//;
+ $string =~ s/\s+$//;
+ return $string;
+}
+
+sub getDeprecated {
+my($jar, $class) = @_;
+
+ open(JAVAP, javap -verbose -public -classpath '$jar' '$class'|);
+
+my $lastMethod = ;
+my %deprecated;
+
+while(JAVAP){
+ chomp();
+if(/^public\s/){
+$lastMethod = $_;
+}
+if(/Deprecated\:\strue/){
+ $lastMethod =~ s/\s+/ /g;
+$deprecated{$lastMethod}=true;
+}
+}
+
+close(JAVAP);
+
+return %deprecated;
+}
+
+sub annotateDeprecated {
+my($jar, $class, $deprecated, $outFile) = @_;
+ open(JAVAP, javap -public -classpath '$jar' '$class'|);
+ open(OUT, $outFile);
+ my @javapOut = JAVAP;
+ @javapOut = sort(@javapOut);
+
+ for my $line (@javapOut){
+ my $trimLine = trim($line);
+ chomp($line);
+ $trimLine =~ s/\s+/ /g;
+ if($deprecated-{$trimLine}){
+ print OUT $line DEPRECATED\n;
+ }else{
+ print OUT $line\n;
+ }
+ }
+
+ close(JAVAP);
+ close(OUT);
+
+}
if(scalar(@ARGV) != 2){
print Usage : diffAPI.pl core jar 1 core jar 2\n;
@@ -24,7 +78,7 @@ if(scalar(@ARGV) != 2){
$jar1 = $ARGV[0];
$jar2 = $ARGV[1];
-$gtCmd = 'egrep accumulo/core/client/.*class|accumulo/core/data/.*class |
grep -v accumulo/core/client/impl | grep -v accumulo/core/data/thrift | tr /
.';
+$gtCmd = 'egrep accumulo/core/client/.*class|accumulo/core/data/.*class |
grep -v accumulo/core/client/impl | grep -v accumulo/core/data/thrift | egrep
-v Impl.*class$ | tr / .';
@classes1 = `jar tf $jar1 | $gtCmd`;
@classes2 = `jar tf $jar2 | $gtCmd`;
@@ -35,12 +89,14 @@ mkdir(diffWorkDir/jar2);
for $class (@classes1){
$class = substr($class, 0, length($class) - 7);
- system(javap -classpath $jar1 $class | sort
diffWorkDir/jar1/$class);
+ %deprecated = getDeprecated($jar1, $class);
+ annotateDeprecated($jar1, $class, \%deprecated,
diffWorkDir/jar1/$class);
}
for $class (@classes2){
$class = substr($class, 0, length($class) - 7);
- system(javap -classpath $jar2 $class | sort
diffWorkDir/jar2/$class);
+ %deprecated = getDeprecated($jar2, $class);
+ annotateDeprecated($jar2, $class, \%deprecated,
diffWorkDir/jar2/$class);
}
system(diff -u diffWorkDir/jar1 diffWorkDir/jar2);
svn commit: r1388669 - in /accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client: ./ impl/
Author: kturner
Date: Fri Sep 21 20:34:54 2012
New Revision: 1388669
URL: http://svn.apache.org/viewvc?rev=1388669view=rev
Log:
ACCUMULO-705 ACCUMULO-706 Made failed attempts to locate a tablet timeout
Added:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TimeoutTabletLocator.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/TimedOutException.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchWriter.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/TimedOutException.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/TimedOutException.java?rev=1388669r1=1388668r2=1388669view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/TimedOutException.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/TimedOutException.java
Fri Sep 21 20:34:54 2012
@@ -43,6 +43,11 @@ public class TimedOutException extends R
}
+ public TimedOutException(String msg) {
+super(msg);
+this.timedoutServers = Collections.emptySet();
+ }
+
public SetString getTimedOutSevers() {
return Collections.unmodifiableSet(timedoutServers);
}
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java?rev=1388669r1=1388668r2=1388669view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java
Fri Sep 21 20:34:54 2012
@@ -199,6 +199,6 @@ public class ScannerOptions implements S
@Override
public long getTimeout(TimeUnit timeunit) {
-return timeOut;
+return timeunit.convert(timeOut, TimeUnit.MILLISECONDS);
}
}
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java?rev=1388669r1=1388668r2=1388669view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
Fri Sep 21 20:34:54 2012
@@ -101,6 +101,8 @@ public class TabletServerBatchReaderIter
private SetString timedoutServers;
private long timeout;
+ private TabletLocator locator;
+
public interface ResultReceiver {
void receive(ListEntryKey,Value entries);
}
@@ -144,6 +146,8 @@ public class TabletServerBatchReaderIter
this.options = new ScannerOptions(scannerOptions);
resultsQueue = new ArrayBlockingQueueListEntryKey,Value(numThreads);
+this.locator = new
TimeoutTabletLocator(TabletLocator.getInstance(instance, credentials, new
Text(table)), timeout);
+
timeoutTrackers = Collections.synchronizedMap(new
HashMapString,TabletServerBatchReaderIterator.TimeoutTracker());
timedoutServers = Collections.synchronizedSet(new HashSetString());
this.timeout = timeout;
@@ -240,7 +244,7 @@ public class TabletServerBatchReaderIter
MapString,MapKeyExtent,ListRange binnedRanges = new
HashMapString,MapKeyExtent,ListRange();
-binRanges(TabletLocator.getInstance(instance, credentials, new
Text(table)), ranges, binnedRanges);
+binRanges(locator, ranges, binnedRanges);
doLookups(binnedRanges, receiver, columns);
}
@@ -314,11 +318,9 @@ public class TabletServerBatchReaderIter
for (ListRange ranges : failures.values())
allRanges.addAll(ranges);
-TabletLocator tabletLocator = TabletLocator.getInstance(instance,
credentials, new Text(table));
-
// since the first call to binRanges clipped the ranges to within a
tablet, we should not get only
// bin to the set of failed tablets
-binRanges(tabletLocator, allRanges, binnedRanges);
+binRanges(locator, allRanges, binnedRanges);
doLookups(binnedRanges, receiver, columns);
}
@@ -360,8 +362,7 @@ public class TabletServerBatchReaderIter
doLookup(tsLocation, tabletsRanges
svn commit: r1388672 - /accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
Author: kturner
Date: Fri Sep 21 20:38:49 2012
New Revision: 1388672
URL: http://svn.apache.org/viewvc?rev=1388672view=rev
Log:
ACCUMULO-705 ACCUMULO-706 removed accidental checkin
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java?rev=1388672r1=1388671r2=1388672view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
Fri Sep 21 20:38:49 2012
@@ -243,8 +243,6 @@ public class ThriftScanner {
if ((currentTime - startTime) / 1000.0 timeOut)
throw new ScanTimedOutException();
- log.trace(((currentTime - startTime) / 1000.0) + + timeOut);
-
Span locateSpan = Trace.start(scan:locateTablet);
try {
loc = TabletLocator.getInstance(instance, credentials,
scanState.tableName).locateTablet(scanState.startRow, scanState.skipStartRow,
false);
svn commit: r1388707 - in /accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands: DeleteCommand.java DeleteManyCommand.java GrepCommand.java InsertCommand.java ScanCommand.jav
Author: kturner
Date: Fri Sep 21 22:16:28 2012
New Revision: 1388707
URL: http://svn.apache.org/viewvc?rev=1388707view=rev
Log:
ACCUMULO-705 ACCUMULO-706 Added timeout option to shell commands that read and
write data
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteCommand.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteManyCommand.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/GrepCommand.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InsertCommand.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteCommand.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteCommand.java?rev=1388707r1=1388706r2=1388707view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteCommand.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteCommand.java
Fri Sep 21 22:16:28 2012
@@ -17,12 +17,14 @@
package org.apache.accumulo.core.util.shell.commands;
import java.io.IOException;
+import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.security.ColumnVisibility;
import
org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
@@ -35,6 +37,15 @@ import org.apache.hadoop.io.Text;
public class DeleteCommand extends Command {
private Option deleteOptAuths, timestampOpt;
+ private Option timeoutOption;
+
+ protected long getTimeout(final CommandLine cl) {
+if (cl.hasOption(timeoutOption.getLongOpt())) {
+ return
AccumuloConfiguration.getTimeInMillis(cl.getOptionValue(timeoutOption.getLongOpt()));
+}
+
+return Long.MAX_VALUE;
+ }
public int execute(final String fullCommand, final CommandLine cl, final
Shell shellState) throws AccumuloException, AccumuloSecurityException,
TableNotFoundException,
IOException, ConstraintViolationException {
@@ -57,7 +68,7 @@ public class DeleteCommand extends Comma
m.putDelete(colf, colq);
}
final BatchWriter bw =
shellState.getConnector().createBatchWriter(shellState.getTableName(),
-new
BatchWriterConfig().setMaxMemory(m.estimatedMemoryUsed()).setMaxWriteThreads(1));
+new
BatchWriterConfig().setMaxMemory(m.estimatedMemoryUsed()).setMaxWriteThreads(1).setTimeout(getTimeout(cl),
TimeUnit.MILLISECONDS));
bw.addMutation(m);
bw.close();
return 0;
@@ -85,6 +96,11 @@ public class DeleteCommand extends Comma
timestampOpt.setArgName(timestamp);
o.addOption(timestampOpt);
+timeoutOption = new Option(null, timeout, true,
+time before insert should fail if no data is written. If no unit is
given assumes seconds. Units d,h,m,s,and ms are supported. e.g. 30s or
100ms);
+timeoutOption.setArgName(timeout);
+o.addOption(timeoutOption);
+
return o;
}
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteManyCommand.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteManyCommand.java?rev=1388707r1=1388706r2=1388707view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteManyCommand.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteManyCommand.java
Fri Sep 21 22:16:28 2012
@@ -16,6 +16,8 @@
*/
package org.apache.accumulo.core.util.shell.commands;
+import java.util.concurrent.TimeUnit;
+
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.IteratorSetting;
@@ -47,11 +49,14 @@ public class DeleteManyCommand extends S
// handle remaining optional arguments
scanner.setRange(getRange(cl, interpeter));
+scanner.setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS);
+
// handle columns
fetchColumns(cl, scanner, interpeter);
// output / delete the records
-final BatchWriter writer =
shellState.getConnector().createBatchWriter
svn commit: r1388710 - /accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InsertCommand.java
Author: kturner
Date: Fri Sep 21 22:31:35 2012
New Revision: 1388710
URL: http://svn.apache.org/viewvc?rev=1388710view=rev
Log:
ACCUMULO-722 made shell insert always print errors
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InsertCommand.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InsertCommand.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InsertCommand.java?rev=1388710r1=1388709r2=1388710view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InsertCommand.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InsertCommand.java
Fri Sep 21 22:31:35 2012
@@ -93,6 +93,14 @@ public class InsertCommand extends Comma
for (ConstraintViolationSummary cvs :
e.getConstraintViolationSummaries()) {
lines.add( + cvs.toString());
}
+
+ if (lines.size() == 0 || e.getUnknownExceptions() 0) {
+// must always print something
+lines.add( + e.getClass().getName() + : + e.getMessage());
+if (e.getCause() != null)
+ lines.add( Caused by : + e.getCause().getClass().getName() +
: + e.getCause().getMessage());
+ }
+
shellState.printLines(lines.iterator(), false);
}
return 0;
svn commit: r1388715 - /accumulo/branches/1.4/bin/start-server.sh
Author: kturner
Date: Fri Sep 21 22:53:06 2012
New Revision: 1388715
URL: http://svn.apache.org/viewvc?rev=1388715view=rev
Log:
ACCUMULO-312 fixed start-server.sh
Modified:
accumulo/branches/1.4/bin/start-server.sh
Modified: accumulo/branches/1.4/bin/start-server.sh
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/bin/start-server.sh?rev=1388715r1=1388714r2=1388715view=diff
==
--- accumulo/branches/1.4/bin/start-server.sh (original)
+++ accumulo/branches/1.4/bin/start-server.sh Fri Sep 21 22:53:06 2012
@@ -39,7 +39,7 @@ SLAVES=`wc -l ${ACCUMULO_HOME}/conf/sl
ip=`ifconfig | grep inet[^6] | awk '{print $2}' | sed 's/addr://' | grep -v
0.0.0.0 | grep -v 127.0.0.1 | head -n 1`
-if [ $HOST == localhost -o $HOST == `hostname` -o $HOST == $ip]
+if [ $HOST == localhost -o $HOST == `hostname` -o $HOST == $ip ]
then
PID=`ps -ef | egrep ${ACCUMULO_HOME}/.*/accumulo.*.jar | grep Main
$SERVICE | grep -v grep | awk {'print $2'} | head -1`
else
@@ -48,7 +48,7 @@ fi
if [ -z $PID ]; then
echo Starting $LONGNAME on $HOST
- if [ $HOST == localhost -o $HOST == `hostname` -o $HOST == $ip]
+ if [ $HOST == localhost -o $HOST == `hostname` -o $HOST == $ip ]
then
${bin}/accumulo ${SERVICE} --address $1
${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out
2${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err
MAX_FILES_OPEN=`bash -c 'ulimit -n'`
svn commit: r1389463 - in /accumulo/trunk/test/compat/japi-compliance: ./ README japi-accumulo-1.4.xml japi-accumulo-1.5.xml
Author: kturner Date: Mon Sep 24 16:10:41 2012 New Revision: 1389463 URL: http://svn.apache.org/viewvc?rev=1389463view=rev Log: ACCUMULO-765 added documentation on how to run japi-compliance Added: accumulo/trunk/test/compat/japi-compliance/ accumulo/trunk/test/compat/japi-compliance/README accumulo/trunk/test/compat/japi-compliance/japi-accumulo-1.4.xml accumulo/trunk/test/compat/japi-compliance/japi-accumulo-1.5.xml Added: accumulo/trunk/test/compat/japi-compliance/README URL: http://svn.apache.org/viewvc/accumulo/trunk/test/compat/japi-compliance/README?rev=1389463view=auto == --- accumulo/trunk/test/compat/japi-compliance/README (added) +++ accumulo/trunk/test/compat/japi-compliance/README Mon Sep 24 16:10:41 2012 @@ -0,0 +1,14 @@ +There is a tool that can analyze the difference between APIs called +japi-compliance[1]. This tool is useful for checking API compatability of +different Accumulo versions. To run this tool edit the xml files to specify +the location of accumulo core jars and set the library version. Then run the +following command. + + japi-compliance-checker.pl -skip-deprecated -old japi-accumulo-1.4.xml -new japi-accumulo-1.5.xml -l accumulo + + +[1]: http://ispras.linuxbase.org/index.php/Java_API_Compliance_Checker + + + + Added: accumulo/trunk/test/compat/japi-compliance/japi-accumulo-1.4.xml URL: http://svn.apache.org/viewvc/accumulo/trunk/test/compat/japi-compliance/japi-accumulo-1.4.xml?rev=1389463view=auto == --- accumulo/trunk/test/compat/japi-compliance/japi-accumulo-1.4.xml (added) +++ accumulo/trunk/test/compat/japi-compliance/japi-accumulo-1.4.xml Mon Sep 24 16:10:41 2012 @@ -0,0 +1,18 @@ +version + 1.4.2-SNAPSHOT +/version + +archives + CHANGEME/accumulo-core-1.4.2-SNAPSHOT.jar +/archives + +skip_packages + org.apache.accumulo.core.client.impl + org.apache.accumulo.core.client.mock + org.apache.accumulo.core.data.thrift +/skip_packages + +packages + org.apache.accumulo.core.client + org.apache.accumulo.core.data +/packages Added: accumulo/trunk/test/compat/japi-compliance/japi-accumulo-1.5.xml URL: http://svn.apache.org/viewvc/accumulo/trunk/test/compat/japi-compliance/japi-accumulo-1.5.xml?rev=1389463view=auto == --- accumulo/trunk/test/compat/japi-compliance/japi-accumulo-1.5.xml (added) +++ accumulo/trunk/test/compat/japi-compliance/japi-accumulo-1.5.xml Mon Sep 24 16:10:41 2012 @@ -0,0 +1,19 @@ +version + 1.5.0-SNAPSHOT +/version + +archives + ../../../lib/accumulo-core-1.5.0-SNAPSHOT.jar +/archives + +skip_packages + org.apache.accumulo.core.client.impl + org.apache.accumulo.core.client.mock + org.apache.accumulo.core.data.thrift +/skip_packages + +packages + org.apache.accumulo.core.client + org.apache.accumulo.core.data +/packages +
svn commit: r1393868 - in /accumulo/branches/1.4: src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvat
Author: kturner
Date: Thu Oct 4 01:27:15 2012
New Revision: 1393868
URL: http://svn.apache.org/viewvc?rev=1393868view=rev
Log:
ACCUMULO-779 fixed Fate starvation bug
Added:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java
accumulo/branches/1.4/test/system/auto/simple/fateStartvation.py (with
props)
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java?rev=1393868r1=1393867r2=1393868view=diff
==
---
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java
(original)
+++
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java
Thu Oct 4 01:27:15 2012
@@ -47,6 +47,7 @@ public class ZooStoreT implements TSto
private String path;
private IZooReaderWriter zk;
+ private String lastReserved = ;
private SetLong reserved;
private MapLong,Long defered;
private SecureRandom idgenerator;
@@ -123,20 +124,33 @@ public class ZooStoreT implements TSto
events = statusChangeEvents;
}
-ListString txdirs = zk.getChildren(path);
+ListString txdirs = new ArrayListString(zk.getChildren(path));
+Collections.sort(txdirs);
+
+synchronized (this) {
+ if (txdirs.size() 0 txdirs.get(txdirs.size() -
1).compareTo(lastReserved) = 0)
+lastReserved = ;
+}
for (String txdir : txdirs) {
long tid = parseTid(txdir);
synchronized (this) {
+// this check makes reserve pick up where it left off, so that it
cycles through all as it is repeatedly called failing to do so can lead to
+// starvation where fate ops that sort higher and hold a lock are
never reserved.
+if (txdir.compareTo(lastReserved) = 0)
+ continue;
+
if (defered.containsKey(tid)) {
if (defered.get(tid) System.currentTimeMillis())
defered.remove(tid);
else
continue;
}
-if (!reserved.contains(tid))
+if (!reserved.contains(tid)) {
reserved.add(tid);
+ lastReserved = txdir;
+}
else
continue;
}
Added:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java?rev=1393868view=auto
==
---
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java
(added)
+++
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java
Thu Oct 4 01:27:15 2012
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.test.functional;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.accumulo.server.test.TestIngest;
+import org.apache.hadoop.io.Text;
+
+/**
+ * See ACCUMULO-779
+ */
+public class FateStarvationTest extends FunctionalTest {
+
+ @Override
+ public void cleanup() throws Exception {}
+
+ @Override
+ public MapString,String getInitialConfig() {
+return Collections.emptyMap();
+ }
+
+ @Override
+ public ListTableSetup getTablesToCreate() {
+return Collections.emptyList();
+ }
+
+ @Override
+ public void run() throws Exception {
+getConnector().tableOperations().create(test_ingest);
+
+getConnector().tableOperations
svn commit: r1393876 - in /accumulo/trunk: ./ core/ fate/src/main/java/org/apache/accumulo/fate/ZooStore.java server/ server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest
Author: kturner
Date: Thu Oct 4 02:00:27 2012
New Revision: 1393876
URL: http://svn.apache.org/viewvc?rev=1393876view=rev
Log:
ACCUMULO-779 fixed Fate starvation bug (merged from 1.4)
Added:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java
- copied unchanged from r1393868,
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java
accumulo/trunk/test/system/auto/simple/fateStartvation.py
- copied unchanged from r1393868,
accumulo/branches/1.4/test/system/auto/simple/fateStartvation.py
Modified:
accumulo/trunk/ (props changed)
accumulo/trunk/core/ (props changed)
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
(contents, props changed)
accumulo/trunk/server/ (props changed)
accumulo/trunk/src/ (props changed)
Propchange: accumulo/trunk/
--
Merged /accumulo/branches/1.4:r1393868
Propchange: accumulo/trunk/core/
--
Merged /accumulo/branches/1.4/core:r1393868
Modified:
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java?rev=1393876r1=1393875r2=1393876view=diff
==
--- accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
(original)
+++ accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
Thu Oct 4 02:00:27 2012
@@ -47,6 +47,7 @@ public class ZooStoreT implements TSto
private String path;
private IZooReaderWriter zk;
+ private String lastReserved = ;
private SetLong reserved;
private MapLong,Long defered;
private SecureRandom idgenerator;
@@ -123,20 +124,33 @@ public class ZooStoreT implements TSto
events = statusChangeEvents;
}
-ListString txdirs = zk.getChildren(path);
+ListString txdirs = new ArrayListString(zk.getChildren(path));
+Collections.sort(txdirs);
+
+synchronized (this) {
+ if (txdirs.size() 0 txdirs.get(txdirs.size() -
1).compareTo(lastReserved) = 0)
+lastReserved = ;
+}
for (String txdir : txdirs) {
long tid = parseTid(txdir);
synchronized (this) {
+// this check makes reserve pick up where it left off, so that it
cycles through all as it is repeatedly called failing to do so can lead to
+// starvation where fate ops that sort higher and hold a lock are
never reserved.
+if (txdir.compareTo(lastReserved) = 0)
+ continue;
+
if (defered.containsKey(tid)) {
if (defered.get(tid) System.currentTimeMillis())
defered.remove(tid);
else
continue;
}
-if (!reserved.contains(tid))
+if (!reserved.contains(tid)) {
reserved.add(tid);
+ lastReserved = txdir;
+}
else
continue;
}
Propchange:
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
--
--- svn:mergeinfo (added)
+++ svn:mergeinfo Thu Oct 4 02:00:27 2012
@@ -0,0 +1,16 @@
+/accumulo/branches/1.3/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java:1309369,1328076,1330246,1330264,1330944,1349971,1354669
+/accumulo/branches/1.3/src/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java:1309369,1328076,1330246,1349971,1354669
+/accumulo/branches/1.4/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java:1305403-1382577,1382613,1388120,1388629,1393868
+/accumulo/branches/1.4/src/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java:1305403-1356900,1358206,1363430,1364778,1365213,1382566,1382923,1388120
+/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java:1393868
+/accumulo/branches/ACCUMULO-672/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java:1357826,1357829,1357842,1357858,1358236,1359163
+/accumulo/trunk/src/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java:1329425,1332224,1332278,1332347,1333047,1333070,1341000,1342373,1350779,1351691,1356400,1359721
+/incubator/accumulo/branches/1.3/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071
svn commit: r1397391 - in /accumulo/trunk: ./ core/ core/src/main/java/org/apache/accumulo/core/client/impl/ core/src/main/java/org/apache/accumulo/core/util/ core/src/test/java/org/apache/accumulo/co
Author: kturner
Date: Fri Oct 12 00:22:14 2012
New Revision: 1397391
URL: http://svn.apache.org/viewvc?rev=1397391view=rev
Log:
ACCUMULO-806 fixed bug where a tablet location could not be found when the
metadata table had empty tablets (merged from 1.4)
Modified:
accumulo/trunk/ (props changed)
accumulo/trunk/core/ (props changed)
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocator.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
(props changed)
accumulo/trunk/server/ (props changed)
accumulo/trunk/src/ (props changed)
Propchange: accumulo/trunk/
--
Merged /accumulo/branches/1.4/src:r1397383
Merged /accumulo/branches/1.4:r1397383
Propchange: accumulo/trunk/core/
--
Merged /accumulo/branches/1.4/core:r1397383
Merged /accumulo/branches/1.4/src/core:r1397383
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java?rev=1397391r1=1397390r2=1397391view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
Fri Oct 12 00:22:14 2012
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.client.A
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocation;
+import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocations;
import
org.apache.accumulo.core.client.impl.TabletLocatorImpl.TabletLocationObtainer;
import
org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator.ResultReceiver;
import org.apache.accumulo.core.data.Column;
@@ -44,6 +45,7 @@ import org.apache.accumulo.core.security
import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
import org.apache.accumulo.core.util.MetadataTable;
import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.TextUtil;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Level;
@@ -68,12 +70,12 @@ public class MetadataLocationObtainer im
}
@Override
- public ListTabletLocation lookupTablet(TabletLocation src, Text row, Text
stopRow, TabletLocator parent) throws AccumuloSecurityException,
+ public TabletLocations lookupTablet(TabletLocation src, Text row, Text
stopRow, TabletLocator parent) throws AccumuloSecurityException,
AccumuloException {
-
-ArrayListTabletLocation list = new ArrayListTabletLocation();
-
+
try {
+ ArrayListTabletLocation list = new ArrayListTabletLocation();
+
OpTimer opTimer = null;
if (log.isTraceEnabled())
opTimer = new OpTimer(log, Level.TRACE).start(Looking up in +
src.tablet_extent.getTableId() + row= + TextUtil.truncate(row) + extent=
@@ -98,12 +100,14 @@ public class MetadataLocationObtainer im
// System.out.println(results +results.keySet());
- SortedMapKeyExtent,Text metadata =
MetadataTable.getMetadataLocationEntries(results);
+ PairSortedMapKeyExtent,Text,ListKeyExtent metadata =
MetadataTable.getMetadataLocationEntries(results);
- for (EntryKeyExtent,Text entry : metadata.entrySet()) {
+ for (EntryKeyExtent,Text entry : metadata.getFirst().entrySet()) {
list.add(new TabletLocation(entry.getKey(),
entry.getValue().toString()));
}
+ return new TabletLocations(list, metadata.getSecond());
+
} catch (AccumuloServerException ase) {
if (log.isTraceEnabled())
log.trace(src.tablet_extent.getTableId() + lookup failed, +
src.tablet_location + server side exception);
@@ -118,7 +122,7 @@ public class MetadataLocationObtainer im
parent.invalidateCache(src.tablet_location);
}
-return list;
+return null;
}
@Override
@@ -161,7 +165,7 @@ public class MetadataLocationObtainer im
throw e;
}
-SortedMapKeyExtent,Text metadata =
MetadataTable.getMetadataLocationEntries(results
svn commit: r1397921 - in /accumulo/branches/1.4/src/core/src: main/java/org/apache/accumulo/core/client/admin/ main/java/org/apache/accumulo/core/client/mock/ main/java/org/apache/accumulo/core/itera
Author: kturner
Date: Sat Oct 13 18:52:18 2012
New Revision: 1397921
URL: http://svn.apache.org/viewvc?rev=1397921view=rev
Log:
ACCUMULO-795 Applied patch from Drew Farris that allows mock to create tables
w/o versioning
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java?rev=1397921r1=1397920r2=1397921view=diff
==
---
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
(original)
+++
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
Sat Oct 13 18:52:18 2012
@@ -180,11 +180,7 @@ public class TableOperationsImpl extends
ListByteBuffer args =
Arrays.asList(ByteBuffer.wrap(tableName.getBytes()),
ByteBuffer.wrap(timeType.name().getBytes()));
-MapString,String opts;
-if (limitVersion) {
- opts = IteratorUtil.generateInitialTableProperties();
-} else
- opts = Collections.emptyMap();
+MapString,String opts =
IteratorUtil.generateInitialTableProperties(limitVersion);
try {
doTableOperation(TableOperation.CREATE, args, opts);
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java?rev=1397921r1=1397920r2=1397921view=diff
==
---
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
(original)
+++
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
Sat Oct 13 18:52:18 2012
@@ -86,9 +86,9 @@ public class MockTable {
MapString,EnumSetTablePermission userPermissions = new
HashMapString,EnumSetTablePermission();
private TimeType timeType;
- MockTable(boolean useVersions, TimeType timeType) {
+ MockTable(boolean limitVersion, TimeType timeType) {
this.timeType = timeType;
-settings = IteratorUtil.generateInitialTableProperties();
+settings = IteratorUtil.generateInitialTableProperties(limitVersion);
for (EntryString,String entry :
AccumuloConfiguration.getDefaultConfiguration()) {
String key = entry.getKey();
if (key.startsWith(Property.TABLE_PREFIX.getKey()))
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java?rev=1397921r1=1397920r2=1397921view=diff
==
---
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
(original)
+++
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
Sat Oct 13 18:52:18 2012
@@ -87,13 +87,22 @@ public class IteratorUtil {
return props;
}
- public static MapString,String generateInitialTableProperties() {
+ /**
+ * Generate the initial (default) properties for a table
+ * @param limitVersion
+ * include a VersioningIterator at priority 20 that retains a single
version of a given K/V pair.
+ * @return A map of Table properties
+ */
+ public static MapString,String generateInitialTableProperties(boolean
limitVersion) {
TreeMapString,String props = new TreeMapString,String();
-for (IteratorScope iterScope : IteratorScope.values()) {
- props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() + .vers,
20, + VersioningIterator.class.getName());
- props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() +
.vers.opt.maxVersions, 1);
+if (limitVersion) {
+for (IteratorScope iterScope : IteratorScope.values()) {
+ props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() +
.vers, 20, + VersioningIterator.class.getName());
+ props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() +
.vers.opt.maxVersions, 1);
+}
}
+
return
svn commit: r1397936 - in /accumulo/trunk: ./ core/ core/src/main/java/org/apache/accumulo/core/client/admin/ core/src/main/java/org/apache/accumulo/core/client/mock/ core/src/main/java/org/apache/acc
Author: kturner
Date: Sat Oct 13 19:43:30 2012
New Revision: 1397936
URL: http://svn.apache.org/viewvc?rev=1397936view=rev
Log:
ACCUMULO-795 Applied patch from Drew Farris that allows mock to create tables
w/o versioning (merged from 1.4)
Modified:
accumulo/trunk/ (props changed)
accumulo/trunk/core/ (props changed)
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
(props changed)
accumulo/trunk/server/ (props changed)
accumulo/trunk/src/ (props changed)
Propchange: accumulo/trunk/
--
Merged /accumulo/branches/1.4/src:r1397921
Merged /accumulo/branches/1.4:r1397921
Propchange: accumulo/trunk/core/
--
Merged /accumulo/branches/1.4/core:r1397921
Merged /accumulo/branches/1.4/src/core:r1397921
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java?rev=1397936r1=1397935r2=1397936view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
Sat Oct 13 19:43:30 2012
@@ -190,11 +190,7 @@ public class TableOperationsImpl extends
ListByteBuffer args =
Arrays.asList(ByteBuffer.wrap(tableName.getBytes()),
ByteBuffer.wrap(timeType.name().getBytes()));
-MapString,String opts;
-if (limitVersion) {
- opts = IteratorUtil.generateInitialTableProperties();
-} else
- opts = Collections.emptyMap();
+MapString,String opts =
IteratorUtil.generateInitialTableProperties(limitVersion);
try {
doTableOperation(TableOperation.CREATE, args, opts);
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java?rev=1397936r1=1397935r2=1397936view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
Sat Oct 13 19:43:30 2012
@@ -88,9 +88,9 @@ public class MockTable {
private TimeType timeType;
SortedSetText splits = new TreeSetText();
- MockTable(boolean useVersions, TimeType timeType) {
+ MockTable(boolean limitVersion, TimeType timeType) {
this.timeType = timeType;
-settings = IteratorUtil.generateInitialTableProperties();
+settings = IteratorUtil.generateInitialTableProperties(limitVersion);
for (EntryString,String entry :
AccumuloConfiguration.getDefaultConfiguration()) {
String key = entry.getKey();
if (key.startsWith(Property.TABLE_PREFIX.getKey()))
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java?rev=1397936r1=1397935r2=1397936view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
Sat Oct 13 19:43:30 2012
@@ -63,13 +63,22 @@ public class IteratorUtil {
}
- public static MapString,String generateInitialTableProperties() {
+ /**
+ * Generate the initial (default) properties for a table
+ * @param limitVersion
+ * include a VersioningIterator at priority 20 that retains a single
version of a given K/V pair.
+ * @return A map of Table properties
+ */
+ public static MapString,String generateInitialTableProperties(boolean
limitVersion) {
TreeMapString,String props = new TreeMapString,String();
-for (IteratorScope iterScope : IteratorScope.values()) {
- props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() + .vers,
20, + VersioningIterator.class.getName());
- props.put
svn commit: r1398451 - in /accumulo/trunk/core/src: main/java/org/apache/accumulo/core/iterators/user/ main/java/org/apache/accumulo/core/util/shell/commands/ test/java/org/apache/accumulo/core/iterat
Author: kturner
Date: Mon Oct 15 18:56:00 2012
New Revision: 1398451
URL: http://svn.apache.org/viewvc?rev=1398451view=rev
Log:
ACCUMULO-720 Applied patch, from Chris Bennight, that adds subsequence matching
to egrep shell command
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/EGrepCommand.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/GrepCommand.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java?rev=1398451r1=1398450r2=1398451view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
Mon Oct 15 18:56:00 2012
@@ -46,12 +46,15 @@ public class RegExFilter extends Filter
return result;
}
+
+
public static final String ROW_REGEX = rowRegex;
public static final String COLF_REGEX = colfRegex;
public static final String COLQ_REGEX = colqRegex;
public static final String VALUE_REGEX = valueRegex;
public static final String OR_FIELDS = orFields;
public static final String ENCODING = encoding;
+ public static final String MATCH_SUBSTRING = matchSubstring;
public static final String ENCODING_DEFAULT = UTF-8;
@@ -60,6 +63,7 @@ public class RegExFilter extends Filter
private Matcher colqMatcher;
private Matcher valueMatcher;
private boolean orFields = false;
+ private boolean matchSubstring = false;
private String encoding = ENCODING_DEFAULT;
@@ -74,7 +78,7 @@ public class RegExFilter extends Filter
if (matcher != null) {
try {
matcher.reset(new String(bs.getBackingArray(), bs.offset(),
bs.length(), encoding));
-return matcher.matches();
+return matchSubstring ? matcher.find() : matcher.matches();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
@@ -86,7 +90,7 @@ public class RegExFilter extends Filter
if (matcher != null) {
try {
matcher.reset(new String(data, offset, len, encoding));
-return matcher.matches();
+return matchSubstring ? matcher.find() : matcher.matches();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
@@ -144,6 +148,12 @@ public class RegExFilter extends Filter
orFields = false;
}
+if (options.containsKey(MATCH_SUBSTRING)) {
+ matchSubstring = Boolean.parseBoolean(options.get(MATCH_SUBSTRING));
+} else {
+ matchSubstring = false;
+}
+
if (options.containsKey(ENCODING)) {
encoding = options.get(ENCODING);
}
@@ -159,6 +169,7 @@ public class RegExFilter extends Filter
io.addNamedOption(RegExFilter.COLQ_REGEX, regular expression on column
qualifier);
io.addNamedOption(RegExFilter.VALUE_REGEX, regular expression on value);
io.addNamedOption(RegExFilter.OR_FIELDS, use OR instread of AND when
multiple regexes given);
+io.addNamedOption(RegExFilter.MATCH_SUBSTRING, match on substrings);
io.addNamedOption(RegExFilter.ENCODING, character encoding of byte array
value (default is + ENCODING_DEFAULT + ));
return io;
}
@@ -194,6 +205,8 @@ public class RegExFilter extends Filter
/**
* Encode the terms to match against in the iterator
+ * Same as calling setRegexs(IteratorSetting si, String rowTerm, String
cfTerm, String cqTerm, String valueTerm, boolean orFields, boolean
matchSubstring)
+ * with matchSubstring set to false
*
* @param si
* ScanIterator config to be updated
@@ -208,7 +221,28 @@ public class RegExFilter extends Filter
* @param orFields
* if true, any of the non-null terms can match to return the entry
*/
- public static void setRegexs(IteratorSetting si, String rowTerm, String
cfTerm, String cqTerm, String valueTerm, boolean orFields) {
+ public static void setRegexs(IteratorSetting si, String rowTerm, String
cfTerm, String cqTerm, String valueTerm, boolean orFields) {
+setRegexs(si, rowTerm, cfTerm, cqTerm, valueTerm, orFields, false);
+ }
+
+ /**
+ * Encode the terms to match against in the iterator
+ *
+ * @param si
+ * ScanIterator config to be updated
+ * @param rowTerm
+ * the pattern to match against the Key's row. Not used if null.
+ * @param cfTerm
+ * the pattern to match against the Key's column family. Not used
if null.
+ * @param cqTerm
svn commit: r1400976 - in /accumulo/branches/1.4/src/core/src: main/java/org/apache/accumulo/core/client/mapreduce/ test/java/org/apache/accumulo/core/client/mapreduce/
Author: kturner
Date: Mon Oct 22 18:08:17 2012
New Revision: 1400976
URL: http://svn.apache.org/viewvc?rev=1400976view=rev
Log:
ACCUMULO-826 ACCUMULO-507 reverted revisions
1397700,1382923,1339308,1339223,1336322. These changes caused map reduce jobs
to fail if the process that started the job exited.
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java?rev=1400976r1=1400975r2=1400976view=diff
==
---
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
(original)
+++
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
Mon Oct 22 18:08:17 2012
@@ -38,13 +38,6 @@ import org.apache.accumulo.core.security
import org.apache.accumulo.core.util.ArgumentChecker;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.OutputCommitter;
@@ -75,7 +68,7 @@ public class AccumuloOutputFormat extend
private static final String OUTPUT_INFO_HAS_BEEN_SET = PREFIX +
.configured;
private static final String INSTANCE_HAS_BEEN_SET = PREFIX +
.instanceConfigured;
private static final String USERNAME = PREFIX + .username;
- private static final String PASSWORD_PATH = PREFIX + .password;
+ private static final String PASSWORD = PREFIX + .password;
private static final String DEFAULT_TABLE_NAME = PREFIX + .defaulttable;
private static final String INSTANCE_NAME = PREFIX + .instanceName;
@@ -135,28 +128,10 @@ public class AccumuloOutputFormat extend
ArgumentChecker.notNull(user, passwd);
conf.set(USERNAME, user);
+conf.set(PASSWORD, new String(Base64.encodeBase64(passwd)));
conf.setBoolean(CREATETABLES, createTables);
if (defaultTable != null)
conf.set(DEFAULT_TABLE_NAME, defaultTable);
-
-try {
- FileSystem fs = FileSystem.get(conf);
- Path file = new Path(fs.getWorkingDirectory(),
conf.get(mapred.job.name) + System.currentTimeMillis() + .pw);
- conf.set(PASSWORD_PATH, file.toString());
- FSDataOutputStream fos = fs.create(file, false);
- fs.setPermission(file, new FsPermission(FsAction.ALL, FsAction.NONE,
FsAction.NONE));
- fs.deleteOnExit(file);
-
- byte[] encodedPw = Base64.encodeBase64(passwd);
- fos.writeInt(encodedPw.length);
- fos.write(encodedPw);
- fos.close();
-
- DistributedCache.addCacheFile(file.toUri(), conf);
-} catch (IOException ioe) {
- throw new RuntimeException(ioe);
-}
-
}
/**
@@ -257,28 +232,21 @@ public class AccumuloOutputFormat extend
}
/**
- * @throws IOException
+ * WARNING: The password is stored in the Configuration and shared with all
MapReduce tasks; It is BASE64 encoded to provide a charset safe conversion to a
+ * string, and is not intended to be secure.
*
* @deprecated Use {@link #getPassword(Configuration)} instead
*/
- protected static byte[] getPassword(JobContext job) throws IOException {
+ protected static byte[] getPassword(JobContext job) {
return getPassword(job.getConfiguration());
}
/**
- * @throws IOException
+ * WARNING: The password is stored in the Configuration and shared with all
MapReduce tasks; It is BASE64 encoded to provide a charset safe conversion to a
+ * string, and is not intended to be secure.
*/
- protected static byte[] getPassword(Configuration conf) throws IOException {
-FileSystem fs = FileSystem.get(conf);
-Path file = new Path(conf.get(PASSWORD_PATH));
-
-FSDataInputStream fdis = fs.open(file);
-int length = fdis.readInt();
-byte[] encodedPassword = new byte[length];
-fdis.read(encodedPassword);
-fdis.close();
-
-return Base64.decodeBase64(encodedPassword);
+ protected static byte[] getPassword(Configuration conf) {
+return Base64.decodeBase64(conf.get
svn commit: r1401013 - /accumulo/branches/1.4/CHANGES
Author: kturner Date: Mon Oct 22 19:09:38 2012 New Revision: 1401013 URL: http://svn.apache.org/viewvc?rev=1401013view=rev Log: ACCUMULO-819 updated changes for 1.4.2 release Modified: accumulo/branches/1.4/CHANGES Modified: accumulo/branches/1.4/CHANGES URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/CHANGES?rev=1401013r1=1401012r2=1401013view=diff == --- accumulo/branches/1.4/CHANGES (original) +++ accumulo/branches/1.4/CHANGES Mon Oct 22 19:09:38 2012 @@ -17,6 +17,7 @@ Release Notes - Apache Accumulo - Versio * [ACCUMULO-683] - Accumulo ignores HDFS max replication configuration * [ACCUMULO-719] - Failure compiling native libraries on Mac OSX 10.7.4 * [ACCUMULO-753] - root tablet reference is not consulted during file gc +* [ACCUMULO-764] - stack overflow when starting tablet server after recent java update to 1.6.0_35 * [ACCUMULO-766] - master is leaking ZooLock objects * [ACCUMULO-768] - Typo in User Manual on MapReduce Analytics * [ACCUMULO-770] - TableLoadBalancer / DefaultLoadBalancer assigns tables unevenly @@ -33,10 +34,11 @@ Release Notes - Apache Accumulo - Versio * [ACCUMULO-805] - stack trace in MockTableOperationsTest * [ACCUMULO-806] - tablet locator fails when the metadata table has empty sections * [ACCUMULO-815] - timing too tight on LargeRowTest functional test +* [ACCUMULO-823] - /bin/start-server.sh: line 40: ifconfig: command not found +* [ACCUMULO-826] - MapReduce over accumlo fails if process that started job is killed ** Improvement * [ACCUMULO-364] - Security Policy file should be an example -* [ACCUMULO-507] - Large amount of ranges can prevent job from kicking off * [ACCUMULO-663] - RegexFilter make unecessary object creations * [ACCUMULO-674] - Convert default log4j to use log4j date formatters * [ACCUMULO-752] - Add support for importDirectory to the mock instance
svn commit: r1409017 - /accumulo/trunk/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
Author: kturner
Date: Tue Nov 13 23:05:14 2012
New Revision: 1409017
URL: http://svn.apache.org/viewvc?rev=1409017view=rev
Log:
ACCUMULO-851 modified mutation to implement documented behavior
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/data/Mutation.java?rev=1409017r1=1409016r2=1409017view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
Tue Nov 13 23:05:14 2012
@@ -27,6 +27,7 @@ import java.util.List;
import org.apache.accumulo.core.data.thrift.TMutation;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.TextUtil;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
@@ -298,6 +299,14 @@ public class Mutation implements Writabl
entries++;
}
+ private void put(CharSequence cf, CharSequence cq, byte[] cv, boolean hasts,
long ts, boolean deleted, byte[] val) {
+put(new Text(cf.toString()), new Text(cq.toString()), cv, hasts, ts,
deleted, val);
+ }
+
+ private void put(CharSequence cf, CharSequence cq, byte[] cv, boolean hasts,
long ts, boolean deleted, CharSequence val) {
+put(cf, cq, cv, hasts, ts, deleted, TextUtil.getBytes(new
Text(val.toString(;
+ }
+
public void put(Text columnFamily, Text columnQualifier, Value value) {
put(columnFamily, columnQualifier, EMPTY_BYTES, false, 0l, false,
value.get());
}
@@ -331,51 +340,51 @@ public class Mutation implements Writabl
}
public void put(CharSequence columnFamily, CharSequence columnQualifier,
Value value) {
-put(columnFamily.toString().getBytes(),
columnQualifier.toString().getBytes(), EMPTY_BYTES, false, 0l, false,
value.get());
+put(columnFamily.toString(), columnQualifier.toString(), EMPTY_BYTES,
false, 0l, false, value.get());
}
public void put(CharSequence columnFamily, CharSequence columnQualifier,
ColumnVisibility columnVisibility, Value value) {
-put(columnFamily.toString().getBytes(),
columnQualifier.toString().getBytes(), columnVisibility.getExpression(), false,
0l, false, value.get());
+put(columnFamily.toString(), columnQualifier.toString(),
columnVisibility.getExpression(), false, 0l, false, value.get());
}
public void put(CharSequence columnFamily, CharSequence columnQualifier,
long timestamp, Value value) {
-put(columnFamily.toString().getBytes(),
columnQualifier.toString().getBytes(), EMPTY_BYTES, true, timestamp, false,
value.get());
+put(columnFamily.toString(), columnQualifier.toString(), EMPTY_BYTES,
true, timestamp, false, value.get());
}
public void put(CharSequence columnFamily, CharSequence columnQualifier,
ColumnVisibility columnVisibility, long timestamp, Value value) {
-put(columnFamily.toString().getBytes(),
columnQualifier.toString().getBytes(), columnVisibility.getExpression(), true,
timestamp, false, value.get());
+put(columnFamily.toString(), columnQualifier.toString(),
columnVisibility.getExpression(), true, timestamp, false, value.get());
}
public void putDelete(CharSequence columnFamily, CharSequence
columnQualifier) {
-put(columnFamily.toString().getBytes(),
columnQualifier.toString().getBytes(), EMPTY_BYTES, false, 0l, true,
EMPTY_BYTES);
+put(columnFamily.toString(), columnQualifier.toString(), EMPTY_BYTES,
false, 0l, true, EMPTY_BYTES);
}
public void putDelete(CharSequence columnFamily, CharSequence
columnQualifier, ColumnVisibility columnVisibility) {
-put(columnFamily.toString().getBytes(),
columnQualifier.toString().getBytes(), columnVisibility.getExpression(), false,
0l, true, EMPTY_BYTES);
+put(columnFamily.toString(), columnQualifier.toString(),
columnVisibility.getExpression(), false, 0l, true, EMPTY_BYTES);
}
public void putDelete(CharSequence columnFamily, CharSequence
columnQualifier, long timestamp) {
-put(columnFamily.toString().getBytes(),
columnQualifier.toString().getBytes(), EMPTY_BYTES, true, timestamp, true,
EMPTY_BYTES);
+put(columnFamily.toString(), columnQualifier.toString(), EMPTY_BYTES,
true, timestamp, true, EMPTY_BYTES);
}
public void putDelete(CharSequence columnFamily, CharSequence
columnQualifier, ColumnVisibility columnVisibility, long timestamp) {
-put(columnFamily.toString().getBytes(),
columnQualifier.toString().getBytes(), columnVisibility.getExpression(), true,
timestamp, true, EMPTY_BYTES);
+put(columnFamily.toString(), columnQualifier.toString
svn commit: r1412374 - in /accumulo/trunk/examples/instamo: ./ src/ src/main/ src/main/java/ src/main/java/org/ src/main/java/org/apache/ src/main/java/org/apache/accumulo/ src/main/java/org/apache/ac
Author: kturner
Date: Thu Nov 22 00:03:42 2012
New Revision: 1412374
URL: http://svn.apache.org/viewvc?rev=1412374view=rev
Log:
ACCUMULO-14 added mini cluster example
Added:
accumulo/trunk/examples/instamo/
accumulo/trunk/examples/instamo/README.md
accumulo/trunk/examples/instamo/pom.xml
accumulo/trunk/examples/instamo/src/
accumulo/trunk/examples/instamo/src/main/
accumulo/trunk/examples/instamo/src/main/java/
accumulo/trunk/examples/instamo/src/main/java/org/
accumulo/trunk/examples/instamo/src/main/java/org/apache/
accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/
accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/
accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/AccumuloApp.java
(with props)
accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java
(with props)
accumulo/trunk/examples/instamo/src/main/resources/
accumulo/trunk/examples/instamo/src/main/resources/log4j.properties
accumulo/trunk/examples/instamo/src/test/
accumulo/trunk/examples/instamo/src/test/java/
accumulo/trunk/examples/instamo/src/test/java/ExampleAccumuloUnitTest.java
(with props)
Added: accumulo/trunk/examples/instamo/README.md
URL:
http://svn.apache.org/viewvc/accumulo/trunk/examples/instamo/README.md?rev=1412374view=auto
==
--- accumulo/trunk/examples/instamo/README.md (added)
+++ accumulo/trunk/examples/instamo/README.md Thu Nov 22 00:03:42 2012
@@ -0,0 +1,27 @@
+Instamo
+===
+
+Introduction
+---
+
+Instamo makes it easy to write some code and run it against a local, transient
+[Accumulo](http://accumulo.apache.org) instance in minutes. No setup or
+installation is required. This is possible if Java and Maven are already
+installed by following the steps below.
+
+```
+vim src/main/java/org/apache/accumulo/instamo/AccumuloApp.java
+mvn package
+```
+
+Map Reduce
+--
+
+Its possible to run local map reduce jobs against the MiniAccumuloCluster
+instance. There is an example of this in the src directory The following
+command will run the map reduce example.
+
+```
+mvn exec:exec
+```
+
Added: accumulo/trunk/examples/instamo/pom.xml
URL:
http://svn.apache.org/viewvc/accumulo/trunk/examples/instamo/pom.xml?rev=1412374view=auto
==
--- accumulo/trunk/examples/instamo/pom.xml (added)
+++ accumulo/trunk/examples/instamo/pom.xml Thu Nov 22 00:03:42 2012
@@ -0,0 +1,77 @@
+?xml version=1.0?
+project xmlns=http://maven.apache.org/POM/4.0.0;
xmlns:xsi=http://www.w3.org/2001/XMLSchema-instance;
xsi:schemaLocation=http://maven.apache.org/POM/4.0.0
http://maven.apache.org/maven-v4_0_0.xsd;
+ modelVersion4.0.0/modelVersion
+ groupIdorg.apache.accumulo/groupId
+ artifactIdinstamo/artifactId
+ packagingjar/packaging
+ version1.5.0-SNAPSHOT/version
+ nameinstamo/name
+ urlhttp://maven.apache.org/url
+ properties
+accumulo.version1.5.0-SNAPSHOT/accumulo.version
+maclassorg.apache.accumulo.instamo.MapReduceExample/maclass
+ /properties
+ dependencies
+dependency
+ groupIdjunit/groupId
+ artifactIdjunit/artifactId
+ version4.11/version
+ scopetest/scope
+/dependency
+dependency
+ groupIdorg.apache.accumulo/groupId
+ artifactIdaccumulo-core/artifactId
+ version${accumulo.version}/version
+/dependency
+dependency
+ groupIdorg.apache.accumulo/groupId
+ artifactIdaccumulo-server/artifactId
+ version${accumulo.version}/version
+/dependency
+dependency
+ groupIdorg.apache.zookeeper/groupId
+ artifactIdzookeeper/artifactId
+ version3.3.4/version
+/dependency
+dependency
+ groupIdorg.apache.hadoop/groupId
+ artifactIdhadoop-core/artifactId
+ version1.0.1/version
+/dependency
+dependency
+ groupIdcommons-io/groupId
+ artifactIdcommons-io/artifactId
+ version2.4/version
+/dependency
+dependency
+ groupIdorg.apache.accumulo/groupId
+ artifactIdaccumulo-test/artifactId
+ version${accumulo.version}/version
+/dependency
+ /dependencies
+ build
+plugins
+ plugin
+artifactIdmaven-compiler-plugin/artifactId
+version2.0.2/version
+configuration
+ source1.6/source
+ target1.6/target
+/configuration
+ /plugin
+ plugin
+groupIdorg.codehaus.mojo/groupId
+artifactIdexec-maven-plugin/artifactId
+version1.2.1/version
+configuration
+ executablejava/executable
+ arguments
+argument-classpath/argument
+classpath/
+argument${maclass}/argument
+ /arguments
+/configuration
+ /plugin
+/plugins
+ /build
+/project
Added:
accumulo/trunk/examples/instamo
svn commit: r1413720 - /accumulo/trunk/examples/instamo/
Author: kturner Date: Mon Nov 26 16:58:00 2012 New Revision: 1413720 URL: http://svn.apache.org/viewvc?rev=1413720view=rev Log: ACCUMULO-14 update svn ignore Modified: accumulo/trunk/examples/instamo/ (props changed) Propchange: accumulo/trunk/examples/instamo/ -- --- svn:ignore (added) +++ svn:ignore Mon Nov 26 16:58:00 2012 @@ -0,0 +1 @@ +target
svn commit: r1426414 - in /accumulo/trunk/core/src: main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java main/java/org/apache/accumulo/core/file/rfile/RFile.java test/java/org/apache/accumulo
Author: kturner
Date: Fri Dec 28 06:31:08 2012
New Revision: 1426414
URL: http://svn.apache.org/viewvc?rev=1426414view=rev
Log:
ACCUMULO-863 reduced block index entry to just one key
Added:
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/BlockIndexTest.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java?rev=1426414r1=1426413r2=1426414view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
Fri Dec 28 06:31:08 2012
@@ -57,15 +57,13 @@ public class BlockIndex {
public static class BlockIndexEntry implements ComparableBlockIndexEntry {
-private Key key;
private Key prevKey;
private int entriesLeft;
private int pos;
-public BlockIndexEntry(int pos, int entriesLeft, Key key, Key prevKey) {
+public BlockIndexEntry(int pos, int entriesLeft, Key prevKey) {
this.pos = pos;
this.entriesLeft = entriesLeft;
- this.key = key;
this.prevKey = prevKey;
}
@@ -73,12 +71,10 @@ public class BlockIndex {
* @param key
*/
public BlockIndexEntry(Key key) {
- this.key = key;
+ this.prevKey = key;
}
-public Key getKey() {
- return key;
-}
+
public int getEntriesLeft() {
return entriesLeft;
@@ -86,12 +82,12 @@ public class BlockIndex {
@Override
public int compareTo(BlockIndexEntry o) {
- return key.compareTo(o.key);
+ return prevKey.compareTo(o.prevKey);
}
@Override
public String toString() {
- return key + + entriesLeft + + pos;
+ return prevKey + + entriesLeft + + pos;
}
public Key getPrevKey() {
@@ -116,18 +112,24 @@ public class BlockIndex {
} else {
// found exact key in index
index = pos;
+ while (index 0) {
+if (blockIndex[index].getPrevKey().equals(startKey))
+ index--;
+else
+ break;
+ }
}
// handle case where multiple keys in block are exactly the same, want to
find the earliest key in the index
while (index - 1 0) {
- if (blockIndex[index].getKey().equals(blockIndex[index - 1].getKey()))
+ if (blockIndex[index].getPrevKey().equals(blockIndex[index -
1].getPrevKey()))
index--;
else
break;
}
-if (index == 0 blockIndex[index].getKey().equals(startKey))
+if (index == 0 blockIndex[index].getPrevKey().equals(startKey))
return null;
BlockIndexEntry bie = blockIndex[index];
@@ -162,7 +164,7 @@ public class BlockIndex {
val.readFields(cacheBlock);
if (count 0 count % interval == 0) {
-index.add(new BlockIndexEntry(pos, indexEntry.getNumEntries() - count,
rk.getKey(), myPrevKey));
+index.add(new BlockIndexEntry(pos, indexEntry.getNumEntries() - count,
myPrevKey));
}
count++;
@@ -172,4 +174,8 @@ public class BlockIndex {
cacheBlock.seek(0);
}
+
+ BlockIndexEntry[] getIndexEntries() {
+return blockIndex;
+ }
}
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java?rev=1426414r1=1426413r2=1426414view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
Fri Dec 28 06:31:08 2012
@@ -751,7 +751,7 @@ public class RFile {
// just consumed one key from the input stream, so subtract
one from entries left
entriesLeft = bie.getEntriesLeft() - 1;
prevKey = new Key(bie.getPrevKey());
-currKey = bie.getKey();
+currKey = tmpRk.getKey();
}
}
}
Added:
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/BlockIndexTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/BlockIndexTest.java?rev=1426414view=auto
==
---
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/BlockIndexTest.java
(added)
+++
accumulo/trunk/core/src/test/java/org/apache
svn commit: r1428571 - /accumulo/trunk/bin/bootstrap_hdfs.sh
Author: kturner
Date: Thu Jan 3 20:14:18 2013
New Revision: 1428571
URL: http://svn.apache.org/viewvc?rev=1428571view=rev
Log:
ACCUMULO-865 applied patch from dave marion with sligh modifications
Added:
accumulo/trunk/bin/bootstrap_hdfs.sh (with props)
Added: accumulo/trunk/bin/bootstrap_hdfs.sh
URL:
http://svn.apache.org/viewvc/accumulo/trunk/bin/bootstrap_hdfs.sh?rev=1428571view=auto
==
--- accumulo/trunk/bin/bootstrap_hdfs.sh (added)
+++ accumulo/trunk/bin/bootstrap_hdfs.sh Thu Jan 3 20:14:18 2013
@@ -0,0 +1,68 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the License); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an AS IS BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+bin=`dirname $0`
+bin=`cd $bin; pwd`
+
+. $bin/config.sh
+
+#
+# Find the system context directory in HDFS
+#
+SYSTEM_CONTEXT_HDFS_DIR=`grep -A1 general.vfs.classpaths
$ACCUMULO_HOME/conf/accumulo-site.xml | tail -1 | perl -pe 's/\s+value//;
s/\/value//; print $ARGV[1]'`
+
+if [ -z $SYSTEM_CONTEXT_HDFS_DIR ]; then
+
+ echo Your accumulo-site.xml file is not set up for the HDFS Classloader.
Please add the following to your accumulo-site.xml file:
+ echo
+ echo property
+ echonamegeneral.vfs.classpaths/name
+ echovaluehdfs://host:port/dir/value
+ echodescriptionlocation of the jars for the default (system)
context/description
+ echo /property
+ exit
+fi
+
+#
+# Create the system context directy in HDFS if it does not exist
+#
+$HADOOP_HOME/bin/hadoop fs -ls $SYSTEM_CONTEXT_HDFS_DIR /dev/null
+if [ $? -ne 0 ]; then
+ $HADOOP_HOME/bin/hadoop fs -mkdir $SYSTEM_CONTEXT_HDFS_DIR /dev/null
+fi
+
+#
+# Replicate to all slaves to avoid network contention on startup
+#
+NUM_SLAVES=`wc -l $ACCUMULO_HOME/conf/slaves | grep -P '^\d+(?= )' -o`
+$HADOOP_HOME/bin/hadoop fs -setrep -R $NUM_SLAVES $SYSTEM_CONTEXT_HDFS_DIR
/dev/null
+
+#
+# Copy all jars in lib to the system context directory
+#
+hadoop fs -moveFromLocal $ACCUMULO_HOME/lib/*.jar $SYSTEM_CONTEXT_HDFS_DIR
/dev/null
+
+#
+# We need two of the jars in lib, copy them back out and remove them from the
system context dir
+#
+hadoop fs -copyToLocal $SYSTEM_CONTEXT_HDFS_DIR/log4j-1.2.16.jar
$ACCUMULO_HOME/lib/. /dev/null
+hadoop fs -rmr $SYSTEM_CONTEXT_HDFS_DIR/log4j-1.2.16.jar /dev/null
+hadoop fs -copyToLocal $SYSTEM_CONTEXT_HDFS_DIR/commons-vfs2-2.0.jar
$ACCUMULO_HOME/lib/. /dev/null
+hadoop fs -rmr $SYSTEM_CONTEXT_HDFS_DIR/commons-vfs2-2.0.jar /dev/null
+hadoop fs -copyToLocal
$SYSTEM_CONTEXT_HDFS_DIR/accumulo-start-${ACCUMULO_VERSION}.jar
$ACCUMULO_HOME/lib/. /dev/null
+hadoop fs -rmr $SYSTEM_CONTEXT_HDFS_DIR/accumulo-start-${ACCUMULO_VERSION}.jar
/dev/null
Propchange: accumulo/trunk/bin/bootstrap_hdfs.sh
--
svn:executable = *
svn commit: r1428577 - /accumulo/trunk/bin/bootstrap_hdfs.sh
Author: kturner Date: Thu Jan 3 20:34:03 2013 New Revision: 1428577 URL: http://svn.apache.org/viewvc?rev=1428577view=rev Log: ACCUMULO-865 adjusted replication in bootstrap script Modified: accumulo/trunk/bin/bootstrap_hdfs.sh Modified: accumulo/trunk/bin/bootstrap_hdfs.sh URL: http://svn.apache.org/viewvc/accumulo/trunk/bin/bootstrap_hdfs.sh?rev=1428577r1=1428576r2=1428577view=diff == --- accumulo/trunk/bin/bootstrap_hdfs.sh (original) +++ accumulo/trunk/bin/bootstrap_hdfs.sh Thu Jan 3 20:34:03 2013 @@ -50,13 +50,21 @@ fi # Replicate to all slaves to avoid network contention on startup # NUM_SLAVES=`wc -l $ACCUMULO_HOME/conf/slaves | grep -P '^\d+(?= )' -o` -$HADOOP_HOME/bin/hadoop fs -setrep -R $NUM_SLAVES $SYSTEM_CONTEXT_HDFS_DIR /dev/null + +#let each datanode service around 50 clients +let REP=$NUM_SLAVES/50 + +if [ $REP -lt 3 ]; then + REP=3 +fi # # Copy all jars in lib to the system context directory # hadoop fs -moveFromLocal $ACCUMULO_HOME/lib/*.jar $SYSTEM_CONTEXT_HDFS_DIR /dev/null +$HADOOP_HOME/bin/hadoop fs -setrep -R $REP $SYSTEM_CONTEXT_HDFS_DIR /dev/null + # # We need two of the jars in lib, copy them back out and remove them from the system context dir #
svn commit: r1428622 - /accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
Author: kturner
Date: Thu Jan 3 21:53:06 2013
New Revision: 1428622
URL: http://svn.apache.org/viewvc?rev=1428622view=rev
Log:
ACCUMULO-708 added warning if nothing is added for a classpath entry and
removed some screwy code I recently added
Modified:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
Modified:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java?rev=1428622r1=1428621r2=1428622view=diff
==
---
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
(original)
+++
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
Thu Jan 3 21:53:06 2013
@@ -61,15 +61,7 @@ public class AccumuloClassLoader {
// accumulo home should be set
SITE_CONF = System.getenv(ACCUMULO_HOME) + /conf/ + configFile;
} else {
- try {
-URL siteUrl =
AccumuloClassLoader.getClassLoader().getResource(configFile);
-if (siteUrl != null)
- SITE_CONF = siteUrl.toString();
-else
- SITE_CONF = null;
- } catch (IOException e) {
-throw new RuntimeException(e);
- }
+ SITE_CONF = null;
}
//Register the shutdown hook
@@ -192,7 +184,11 @@ public class AccumuloClassLoader {
if (extJars != null extJars.length 0) {
for (File jar : extJars)
urls.add(jar.toURI().toURL());
+ } else {
+log.warn(ignoring classpath entry + classpath);
}
+} else {
+ log.warn(ignoring classpath entry + classpath);
}
}
} else {
svn commit: r1429150 - in /accumulo/trunk/start/src: main/java/org/apache/accumulo/start/classloader/vfs/ test/java/org/apache/accumulo/start/classloader/vfs/
Author: kturner
Date: Fri Jan 4 22:01:12 2013
New Revision: 1429150
URL: http://svn.apache.org/viewvc?rev=1429150view=rev
Log:
ACCUMULO-930 moved indirection for reloading of classloaders to a higher level
Added:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ReloadingClassLoader.java
Modified:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
Modified:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java?rev=1429150r1=1429149r2=1429150view=diff
==
---
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
(original)
+++
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
Fri Jan 4 22:01:12 2013
@@ -16,12 +16,7 @@
*/
package org.apache.accumulo.start.classloader.vfs;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.security.SecureClassLoader;
import java.util.ArrayList;
-import java.util.Enumeration;
import org.apache.commons.vfs2.FileChangeEvent;
import org.apache.commons.vfs2.FileListener;
@@ -38,7 +33,7 @@ import org.apache.log4j.Logger;
* the delegate object if there is any change in the classpath.
*
*/
-public class AccumuloReloadingVFSClassLoader extends SecureClassLoader
implements FileListener {
+public class AccumuloReloadingVFSClassLoader implements FileListener,
ReloadingClassLoader {
private static final Logger log =
Logger.getLogger(AccumuloReloadingVFSClassLoader.class);
@@ -48,20 +43,16 @@ public class AccumuloReloadingVFSClassLo
private String uris;
private FileObject[] files;
private FileSystemManager vfs = null;
- private ClassLoader parent = null;
+ private ReloadingClassLoader parent = null;
private DefaultFileMonitor monitor = null;
- private volatile VFSClassLoader cl = null;
+ private VFSClassLoader cl = null;
- private synchronized VFSClassLoader getClassloader() {
-if (cl == null) {
+ @Override
+ public synchronized ClassLoader getClassLoader() {
+if (cl == null || cl.getParent() != parent.getClassLoader()) {
try {
files = AccumuloVFSClassLoader.resolve(vfs, uris);
-
-if (null != parent)
- setClassloader(new VFSClassLoader(files, vfs, parent));
-else
- setClassloader(new VFSClassLoader(files, vfs));
-
+setClassloader(new VFSClassLoader(files, vfs,
parent.getClassLoader()));
} catch (FileSystemException fse) {
throw new RuntimeException(fse);
}
@@ -75,9 +66,8 @@ public class AccumuloReloadingVFSClassLo
}
- public AccumuloReloadingVFSClassLoader(String uris, FileSystemManager vfs,
ClassLoader parent, long monitorDelay) throws FileSystemException {
-super(parent);
-
+ public AccumuloReloadingVFSClassLoader(String uris, FileSystemManager vfs,
ReloadingClassLoader parent, long monitorDelay) throws FileSystemException {
+
this.uris = uris;
this.vfs = vfs;
this.parent = parent;
@@ -85,10 +75,7 @@ public class AccumuloReloadingVFSClassLo
ArrayListFileObject pathsToMonitor = new ArrayListFileObject();
files = AccumuloVFSClassLoader.resolve(vfs, uris, pathsToMonitor);
-if (null != parent)
- cl = new VFSClassLoader(files, vfs, parent);
-else
- cl = new VFSClassLoader(files, vfs);
+cl = new VFSClassLoader(files, vfs, parent.getClassLoader());
monitor = new DefaultFileMonitor(this);
monitor.setDelay(monitorDelay);
@@ -100,7 +87,7 @@ public class AccumuloReloadingVFSClassLo
monitor.start();
}
- public AccumuloReloadingVFSClassLoader(String uris, FileSystemManager vfs,
ClassLoader parent) throws FileSystemException {
+ public AccumuloReloadingVFSClassLoader(String uris, FileSystemManager vfs,
final ReloadingClassLoader parent) throws FileSystemException {
this(uris, vfs, parent, DEFAULT_TIMEOUT);
}
@@ -133,55 +120,7 @@ public class AccumuloReloadingVFSClassLo
setClassloader(null);
}
- @Override
- public Class? loadClass(String
svn commit: r1430482 - /accumulo/site/trunk/content/people.mdtext
Author: kturner
Date: Tue Jan 8 20:18:13 2013
New Revision: 1430482
URL: http://svn.apache.org/viewvc?rev=1430482view=rev
Log:
CMS commit to accumulo by kturner
Modified:
accumulo/site/trunk/content/people.mdtext
Modified: accumulo/site/trunk/content/people.mdtext
URL:
http://svn.apache.org/viewvc/accumulo/site/trunk/content/people.mdtext?rev=1430482r1=1430481r2=1430482view=diff
==
--- accumulo/site/trunk/content/people.mdtext (original)
+++ accumulo/site/trunk/content/people.mdtext Tue Jan 8 20:18:13 2013
@@ -43,7 +43,7 @@ tr.contributor {
tr class=pmctdacordova/tdtdAaron Cordova/tdtdpmc,
committer/tdtd/td/tr
tr class=pmctdafuchs/tdtdAdam Fuchs/tdtdpmc,
committer/tdtd/td/tr
tr class=pmctdbimargulies/tdtdBenson Margulies/tdtdpmc,
committer/tdtd/td/tr
- tr class=pmctdbillie/tdtdBillie Rinaldi/tdtdpmc,
committer/tdtdHortonworks/td/tr
+ tr class=pmctdbillie/tdtdBillie Rinaldi/tdtdpmc,
committer/tdtda href=http://hortonworks.com/Hortonworks/a/td/tr
tr class=contributortd/tdtdChris
Bennight/tdtdcontributor/tdtd/td/tr
tr class=contributortd/tdtdChris
McCubbin/tdtdcontributor/tdtd/td/tr
tr class=pmctdcawaring/tdtdChris Waring/tdtdpmc,
committer/tdtd/td/tr
@@ -53,9 +53,9 @@ tr.contributor {
tr class=pmctdmedined/tdtdDavid Medinets/tdtdpmc,
committer/tdtd/td/tr
tr class=contributortd/tdtdDennis
Patrone/tdtdcontributor/tdtd/td/tr
tr class=pmctddrew/tdtdDrew Farris/tdtdpmc,
committer/tdtd/td/tr
- tr class=contributortd/tdtdEd
Kohlwey/tdtdcontributor/tdtdBooz Allen Hamilton/td/tr
+ tr class=contributortd/tdtdEd
Kohlwey/tdtdcontributor/tdtda href=http://www.boozallen.com/Booz
Allen Hamilton/td/a/tr
tr class=contributortdedwardyoon/tdtdEdward
Yoon/tdtdcontributor/tdtd/td/tr
- tr class=pmctdecn/tdtdEric Newton/tdtdpmc,
committer/tdtdSW Complete, Inc./td/tr
+ tr class=pmctdecn/tdtdEric Newton/tdtdpmc,
committer/tdtda href=http://swcomplete.com/SW Complete Inc./a/td/tr
tr class=pmctdjtrost/tdtdJason Trost/tdtdpmc,
committer/tdtdEndgame/td/tr
tr class=contributortd/tdtdJay
Shipper/tdtdcontributor/tdtd/td/tr
tr class=contributortd/tdtdJesse
Yates/tdtdcontributor/tdtd/td/tr
@@ -63,7 +63,7 @@ tr.contributor {
tr class=contributortd/tdtdJoe
Skora/tdtdcontributor/tdtd/td/tr
tr class=pmctdvines/tdtdJohn Vines/tdtdpmc,
committer/tdtd/td/tr
tr class=pmctdelserj/tdtdJosh Elser/tdtdpmc,
committer/tdtd/td/tr
- tr class=pmctdkturner/tdtdKeith Turner/tdtdpmc,
committer/tdtd/td/tr
+ tr class=pmctdkturner/tdtdKeith Turner/tdtdpmc,
committer/tdtda href=http://www.ptech-llc.com/Peterson
Technologies/a/td/tr
tr class=contributortd/tdtdLaura
Peaslee/tdtdcontributor/tdtd/td/tr
tr class=contributortd/tdtdLuke
Brassard/tdtdcontributor/tdtd/td/tr
tr class=contributortd/tdtdMatthew
Kirkley/tdtdcontributor/tdtd/td/tr
@@ -74,4 +74,4 @@ tr.contributor {
tr class=contributortd/tdtdTim
Halloran/tdtdcontributor/tdtd/td/tr
tr class=contributortd/tdtdTravis
Pinney/tdtdcontributor/tdtd/td/tr
tr class=pmctdujustgotbilld/tdtdWilliam Slacum/tdtdpmc,
committer/tdtd/td/tr
-/table
+/table
\ No newline at end of file
svn commit: r845573 - in /websites/production/accumulo/content: ./ 1.4/apidocs/
Author: kturner Date: Tue Jan 8 20:20:37 2013 New Revision: 845573 Log: Publishing svnmucc operation to accumulo site by kturner Added: websites/production/accumulo/content/ - copied from r845572, websites/staging/accumulo/trunk/content/ websites/production/accumulo/content/1.4/apidocs/ - copied from r845572, websites/production/accumulo/content/1.4/apidocs/
svn commit: r1430526 - /accumulo/site/trunk/content/people.mdtext
Author: kturner
Date: Tue Jan 8 20:55:26 2013
New Revision: 1430526
URL: http://svn.apache.org/viewvc?rev=1430526view=rev
Log:
CMS commit to accumulo by kturner
Modified:
accumulo/site/trunk/content/people.mdtext
Modified: accumulo/site/trunk/content/people.mdtext
URL:
http://svn.apache.org/viewvc/accumulo/site/trunk/content/people.mdtext?rev=1430526r1=1430525r2=1430526view=diff
==
--- accumulo/site/trunk/content/people.mdtext (original)
+++ accumulo/site/trunk/content/people.mdtext Tue Jan 8 20:55:26 2013
@@ -31,10 +31,10 @@ tr.pmconly {
tr.pmc {
background: #FFF0FF;
}
-tr.committer {
- background: #F0F0FF;
+tr.odd {
+ background: #FF;
}
-tr.contributor {
+tr.even {
background: #F0FFF0;
}
/style
@@ -44,21 +44,21 @@ PMC and Committers
table
trthusername/ththname/ththorganization/ththtimezone/th/tr
- tr class=pmctdacordova/tdtdAaron
Cordova/tdtd/tdtd/td/tr
- tr class=pmctdafuchs/tdtdAdam Fuchs/tdtda
href=http://sqrrl.com/sqrrl/a/tdtd/td/tr
- tr class=pmctdbimargulies/tdtdBenson
Margulies/tdtd/tdtd/td/tr
- tr class=pmctdbillie/tdtdBillie Rinaldi/tdtda
href=http://hortonworks.com/Hortonworks/a/tdtd/td/tr
- tr class=pmctdcawaring/tdtdChris Waring/tdtd/tdtd/td/tr
- tr class=pmctdctubbsii/tdtdChristopher
Tubbs/tdtd/tdtd/td/tr
- tr class=pmctddlmarion/tdtdDave Marion/tdtd/tdtd/td/tr
- tr class=pmctdmedined/tdtdDavid
Medinets/tdtd/tdtd/td/tr
- tr class=pmctddrew/tdtdDrew Farris/tdtd/tdtd/td/tr
- tr class=pmctdecn/tdtdEric Newton/tdtda
href=http://swcomplete.com/SW Complete Inc./a/tdtd-5/td/tr
- tr class=pmctdjtrost/tdtdJason
Trost/tdtdEndgame/tdtd/td/tr
- tr class=pmctdvines/tdtdJohn Vines/tdtda
href=http://sqrrl.com/sqrrl/a/tdtd/td/tr
- tr class=pmctdelserj/tdtdJosh Elser/tdtd/tdtd/td/tr
- tr class=pmctdkturner/tdtdKeith Turner/tdtda
href=http://www.ptech-llc.com/Peterson Technologies/a/tdtd-5/td/tr
- tr class=pmctdujustgotbilld/tdtdWilliam
Slacum/tdtd/tdtd/td/tr
+ tr class=eventdacordova/tdtdAaron
Cordova/tdtd/tdtd/td/tr
+ tr class=oddtdafuchs/tdtdAdam Fuchs/tdtda
href=http://sqrrl.com/sqrrl/a/tdtd/td/tr
+ tr class=eventdbimargulies/tdtdBenson
Margulies/tdtd/tdtd/td/tr
+ tr class=oddtdbillie/tdtdBillie Rinaldi/tdtda
href=http://hortonworks.com/Hortonworks/a/tdtd/td/tr
+ tr class=eventdcawaring/tdtdChris
Waring/tdtd/tdtd/td/tr
+ tr class=oddtdctubbsii/tdtdChristopher
Tubbs/tdtd/tdtd/td/tr
+ tr class=eventddlmarion/tdtdDave Marion/tdtd/tdtd/td/tr
+ tr class=oddtdmedined/tdtdDavid
Medinets/tdtd/tdtd/td/tr
+ tr class=eventddrew/tdtdDrew Farris/tdtd/tdtd/td/tr
+ tr class=oddtdecn/tdtdEric Newton/tdtda
href=http://swcomplete.com/SW Complete Inc./a/tdtd-5/td/tr
+ tr class=eventdjtrost/tdtdJason
Trost/tdtdEndgame/tdtd/td/tr
+ tr class=oddtdvines/tdtdJohn Vines/tdtda
href=http://sqrrl.com/sqrrl/a/tdtd/td/tr
+ tr class=eventdelserj/tdtdJosh Elser/tdtd/tdtd/td/tr
+ tr class=oddtdkturner/tdtdKeith Turner/tdtda
href=http://www.ptech-llc.com/Peterson Technologies/a/tdtd-5/td/tr
+ tr class=eventdujustgotbilld/tdtdWilliam
Slacum/tdtd/tdtd/td/tr
/table
Contributors
@@ -66,23 +66,23 @@ Contributors
table
trthusername/ththname/ththorganization/ththtimezone/th/tr
- tr class=contributortd/tdtdChris
Bennight/tdtd/tdtd/td/tr
- tr class=contributortd/tdtdChris McCubbin/tdtda
href=http://sqrrl.com/sqrrl/a/tdtd/td/tr
- tr class=contributortd/tdtdChristian
Rohling/tdtd/tdtd/td/tr
- tr class=contributortd/tdtdDennis
Patrone/tdtd/tdtd/td/tr
- tr class=contributortd/tdtdEd Kohlwey/tdtda
href=http://www.boozallen.com/Booz Allen Hamilton/td/atd/td/tr
- tr class=contributortdedwardyoon/tdtdEdward
Yoon/tdtd/tdtd/td/tr
- tr class=contributortd/tdtdJay Shipper/tdtd/tdtd/td/tr
- tr class=contributortd/tdtdJesse Yates/tdtd/tdtd/td/tr
- tr class=contributortd/tdtdJim Klucar/tdtd/tdtd/td/tr
- tr class=contributortd/tdtdJoe Skora/tdtd/tdtd/td/tr
- tr class=contributortd/tdtdLaura
Peaslee/tdtd/tdtd/td/tr
- tr class=contributortd/tdtdLuke Brassard/tdtda
href=http://sqrrl.com/sqrrl/a/tdtd/td/tr
- tr class=contributortd/tdtdMatthew
Kirkley/tdtd/tdtd/td/tr
- tr class=contributortd/tdtdMichael Wall/tdtd/tdtd/td/tr
- tr class=contributortd/tdtdPhil Eberhardt/tdtda
href=http://sqrrl.com/sqrrl/a/tdtd/td/tr
- tr class=contributortd/tdtdSapah Shah/tdtd/tdtd/td/tr
- tr class=contributortd/tdtdScott Kuehn/tdtd/tdtd/td/tr
- tr class=contributortd/tdtdTim Halloran/tdtd/tdtd/td/tr
- tr class=contributortd/tdtdTravis
Pinney/tdtd/tdtd/td/tr
+ tr class=eventd/tdtdChris Bennight/tdtd/tdtd/td/tr
+ tr class=oddtd/tdtdChris McCubbin/tdtda
href=http://sqrrl.com/sqrrl/a/tdtd/td/tr
+ tr class=eventd/tdtdChristian Rohling/tdtd/tdtd/td/tr
+ tr class=oddtd/tdtdDennis Patrone/tdtd/tdtd/td/tr
+ tr class=eventd/tdtdEd Kohlwey/tdtda
href=http://www.boozallen.com/Booz Allen Hamilton/td/atd/td/tr
+ tr class=oddtdedwardyoon/tdtdEdward
Yoon/tdtd/tdtd/td/tr
+ tr class=eventd
svn commit: r845590 - in /websites/production/accumulo/content: ./ 1.4/apidocs/
Author: kturner Date: Tue Jan 8 20:57:16 2013 New Revision: 845590 Log: Publishing svnmucc operation to accumulo site by kturner Added: websites/production/accumulo/content/ - copied from r845589, websites/staging/accumulo/trunk/content/ websites/production/accumulo/content/1.4/apidocs/ - copied from r845589, websites/production/accumulo/content/1.4/apidocs/
svn commit: r1431079 - in /accumulo/trunk/start/src: main/java/org/apache/accumulo/start/classloader/vfs/ main/java/org/apache/accumulo/start/classloader/vfs/providers/ main/java/org/apache/commons/vf
Author: kturner
Date: Wed Jan 9 21:11:26 2013
New Revision: 1431079
URL: http://svn.apache.org/viewvc?rev=1431079view=rev
Log:
ACCUMULO-927 applied patch from Dave Marion with modifications
Added:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileAttributes.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileContentInfoFactory.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileProvider.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsRandomAccessContent.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/package.html
accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/
accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/
accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/hdfs/
Removed:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsReadOnlyFileContentInfoFactory.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsReadOnlyRandomAccessContent.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/ReadOnlyHdfsFileProvider.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/ReadOnlyHdfsFileSystem.java
Modified:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileObject.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystemConfigBuilder.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/ReadOnlyHdfsFileProviderTest.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/test/AccumuloDFSBase.java
Modified:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java?rev=1431079r1=1431078r2=1431079view=diff
==
---
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
(original)
+++
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
Wed Jan 9 21:11:26 2013
@@ -23,7 +23,7 @@ import java.util.ArrayList;
import java.util.Collections;
import org.apache.accumulo.start.classloader.AccumuloClassLoader;
-import
org.apache.accumulo.start.classloader.vfs.providers.ReadOnlyHdfsFileProvider;
+import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider;
import org.apache.commons.vfs2.CacheStrategy;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
@@ -208,7 +208,7 @@ public class AccumuloVFSClassLoader {
vfs.addProvider(tbz2, new
org.apache.commons.vfs2.provider.tar.TarFileProvider());
vfs.addProvider(tgz, new
org.apache.commons.vfs2.provider.tar.TarFileProvider());
vfs.addProvider(bz2, new
org.apache.commons.vfs2.provider.bzip2.Bzip2FileProvider());
-vfs.addProvider(hdfs, new ReadOnlyHdfsFileProvider());
+vfs.addProvider(hdfs, new HdfsFileProvider());
vfs.addExtensionMap(jar, jar);
vfs.addExtensionMap(zip, zip);
vfs.addExtensionMap(gz, gz);
Added:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileAttributes.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileAttributes.java?rev=1431079view=auto
==
---
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileAttributes.java
(added)
+++
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileAttributes.java
Wed Jan 9 21:11:26 2013
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org
svn commit: r1431579 - in /accumulo/trunk/start/src/test: java/org/apache/accumulo/start/classloader/vfs/ resources/default/ resources/disabled/
Author: kturner
Date: Thu Jan 10 19:47:20 2013
New Revision: 1431579
URL: http://svn.apache.org/viewvc?rev=1431579view=rev
Log:
ACCUMULO-866 made some of the new VFS classloader test use local filesystem
instead of HDFS. This took about 20 to 30 seconds of the time it takes to
package accumulo-start on my machine.
Removed:
accumulo/trunk/start/src/test/resources/default/
accumulo/trunk/start/src/test/resources/disabled/
Modified:
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
Modified:
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java?rev=1431579r1=1431578r2=1431579view=diff
==
---
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
(original)
+++
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
Thu Jan 10 19:47:20 2013
@@ -16,40 +16,34 @@
*/
package org.apache.accumulo.start.classloader.vfs;
-import java.net.URL;
+import java.io.File;
-import org.apache.accumulo.test.AccumuloDFSBase;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
import org.apache.commons.vfs2.impl.VFSClassLoader;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
-public class AccumuloReloadingVFSClassLoaderTest extends AccumuloDFSBase {
+public class AccumuloReloadingVFSClassLoaderTest {
- private static final Path TEST_DIR = new Path(HDFS_URI + /test-dir);
-
- private FileSystem hdfs = null;
+ private TemporaryFolder folder1 = new TemporaryFolder();
+ private DefaultFileSystemManager vfs;
@Before
public void setup() throws Exception {
Logger.getRootLogger().setLevel(Level.ERROR);
-this.hdfs = cluster.getFileSystem();
-this.hdfs.mkdirs(TEST_DIR);
-
-//Copy jar file to TEST_DIR
-URL jarPath = this.getClass().getResource(/HelloWorld.jar);
-Path src = new Path(jarPath.toURI().toString());
-Path dst = new Path(TEST_DIR, src.getName());
-this.hdfs.copyFromLocalFile(src, dst);
+vfs = ContextManagerTest.getVFS();
+folder1.create();
+FileUtils.copyURLToFile(this.getClass().getResource(/HelloWorld.jar),
folder1.newFile(HelloWorld.jar));
}
FileObject[] createFileSystems(FileObject[] fos) throws FileSystemException {
@@ -66,10 +60,10 @@ public class AccumuloReloadingVFSClassLo
@Test
public void testConstructor() throws Exception {
-FileObject testDir = vfs.resolveFile(TEST_DIR.toUri().toString());
+FileObject testDir = vfs.resolveFile(folder1.getRoot().toURI().toString());
FileObject[] dirContents = testDir.getChildren();
-AccumuloReloadingVFSClassLoader arvcl = new
AccumuloReloadingVFSClassLoader(TEST_DIR.toUri().toString(), vfs, new
ReloadingClassLoader() {
+AccumuloReloadingVFSClassLoader arvcl = new
AccumuloReloadingVFSClassLoader(folder1.getRoot().toURI().toString(), vfs, new
ReloadingClassLoader() {
@Override
public ClassLoader getClassLoader() {
return ClassLoader.getSystemClassLoader();
@@ -86,10 +80,10 @@ public class AccumuloReloadingVFSClassLo
@Test
public void testReloading() throws Exception {
-FileObject testDir = vfs.resolveFile(TEST_DIR.toUri().toString());
+FileObject testDir = vfs.resolveFile(folder1.getRoot().toURI().toString());
FileObject[] dirContents = testDir.getChildren();
-AccumuloReloadingVFSClassLoader arvcl = new
AccumuloReloadingVFSClassLoader(TEST_DIR.toUri().toString(), vfs, new
ReloadingClassLoader() {
+AccumuloReloadingVFSClassLoader arvcl = new
AccumuloReloadingVFSClassLoader(folder1.getRoot().toURI().toString(), vfs, new
ReloadingClassLoader() {
@Override
public ClassLoader getClassLoader() {
return ClassLoader.getSystemClassLoader();
@@ -107,11 +101,10 @@ public class AccumuloReloadingVFSClassLo
Class? clazz1_5 = arvcl.getClassLoader().loadClass(test.HelloWorld);
Assert.assertEquals(clazz1, clazz1_5);
-//Update the class
-URL jarPath = this.getClass
svn commit: r1432111 - in /accumulo/trunk/test/src: main/java/org/apache/accumulo/test/MiniAccumuloCluster.java test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java
Author: kturner
Date: Fri Jan 11 15:56:03 2013
New Revision: 1432111
URL: http://svn.apache.org/viewvc?rev=1432111view=rev
Log:
ACCUMULO-14 fixed issue with starting more than one mini accumulo cluster from
the same jvm
Modified:
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java
Modified:
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java?rev=1432111r1=1432110r2=1432111view=diff
==
---
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
(original)
+++
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
Fri Jan 11 15:56:03 2013
@@ -35,13 +35,10 @@ import java.util.TimerTask;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.master.Master;
import org.apache.accumulo.server.tabletserver.TabletServer;
import org.apache.accumulo.server.util.Initialize;
import org.apache.accumulo.server.util.time.SimpleTimer;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
import org.apache.zookeeper.server.ZooKeeperServerMain;
/**
@@ -185,21 +182,6 @@ public class MiniAccumuloCluster {
return process;
}
- private void init() {
-
-Level prevLevel = Logger.getLogger(org.apache).getLevel();
-Logger.getLogger(org.apache).setLevel(Level.ERROR);
-
-ServerConfiguration.getSiteConfiguration().set(Property.INSTANCE_DFS_URI,
file:///);
-ServerConfiguration.getSiteConfiguration().set(Property.INSTANCE_DFS_DIR,
accumuloDir.getAbsolutePath());
-ServerConfiguration.getSiteConfiguration().set(Property.INSTANCE_ZK_HOST,
localhost: + zooKeeperPort);
-ServerConfiguration.getSiteConfiguration().set(Property.INSTANCE_SECRET,
INSTANCE_SECRET);
-
-Initialize.main(new String[] {--instance-name, INSTANCE_NAME,
--password, rootPassword});
-
-Logger.getLogger(org.apache).setLevel(prevLevel);
- }
-
private void appendProp(FileWriter fileWriter, Property key, String value,
MapString,String siteConfig) throws IOException {
appendProp(fileWriter, key.getKey(), value, siteConfig);
}
@@ -323,7 +305,11 @@ public class MiniAccumuloCluster {
// sleep a little bit to let zookeeper come up before calling init, seems
to work better
UtilWaitThread.sleep(250);
-init();
+Process initProcess = exec(Initialize.class, --instance-name,
INSTANCE_NAME, --password, rootPassword);
+int ret = initProcess.waitFor();
+if (ret != 0) {
+ throw new RuntimeException(Initialize process returned + ret);
+}
tabletServerProcess = exec(TabletServer.class);
masterProcess = exec(Master.class);
Modified:
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java?rev=1432111r1=1432110r2=1432111view=diff
==
---
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java
(original)
+++
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java
Fri Jan 11 15:56:03 2013
@@ -38,6 +38,8 @@ import org.apache.accumulo.core.security
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -56,6 +58,8 @@ public class MiniAccumuloClusterTest {
folder.create();
+Logger.getLogger(org.apache.zookeeper).setLevel(Level.WARN);
+
accumulo = new MiniAccumuloCluster(folder.getRoot(), superSecret, new
HashMapString,String());
accumulo.start();
svn commit: r1432301 - /accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstance.java
Author: kturner
Date: Fri Jan 11 21:53:40 2013
New Revision: 1432301
URL: http://svn.apache.org/viewvc?rev=1432301view=rev
Log:
ACCUMULO-878 added some documentation for new users to mock accumulo about mini
accumulo cluster
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstance.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstance.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstance.java?rev=1432301r1=1432300r2=1432301view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstance.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstance.java
Fri Jan 11 21:53:40 2013
@@ -37,6 +37,17 @@ import org.apache.accumulo.core.util.Tex
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.Text;
+/**
+ * Mock Accumulo provides an in memory inmplementation of the Accumulo client
API. Its possible that the behavior of this implementation may differ from the
+ * behavior of Accumulo. This could result in unit test that pass on Mock
Accumulo and fail on Accumulo or visa-vera. Documenting the differences would be
+ * difficult and is not done.
+ *
+ * p
+ * An alternative to Mock Accumulo called MiniAccumuloCluster was introduced
in Accumulo 1.5. MiniAccumuloCluster spins up actual Accumulo server processes,
can
+ * be used for unit testing, and its behavior should match Accumulo. The
drawback of MiniAccumuloCluster is that its much slower than Mock Accumulo.
+ *
+ */
+
public class MockInstance implements Instance {
static final String genericAddress = localhost:1234;
svn commit: r1433170 - /accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/GenSplits.java
Author: kturner
Date: Mon Jan 14 22:12:08 2013
New Revision: 1433170
URL: http://svn.apache.org/viewvc?rev=1433170view=rev
Log:
ACCUMULO-515 added min and max parameters to GenSplits
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/GenSplits.java
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/GenSplits.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/GenSplits.java?rev=1433170r1=1433169r2=1433170view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/GenSplits.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/GenSplits.java
Mon Jan 14 22:12:08 2013
@@ -16,30 +16,65 @@
*/
package org.apache.accumulo.server.test.continuous;
+import java.util.List;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterException;
+
/**
*
*/
public class GenSplits {
+
+ static class Opts {
+@Parameter(names = --min, description = minimum row)
+long minRow = 0;
+
+@Parameter(names = --max, description = maximum row)
+long maxRow = Long.MAX_VALUE;
+
+@Parameter(description = num tablets)
+ListString args = null;
+ }
+
public static void main(String[] args) {
-if (args.length != 1) {
- System.err.println(Usage: + GenSplits.class.getName() + num
tablets);
+Opts opts = new Opts();
+JCommander jcommander = new JCommander(opts);
+jcommander.setProgramName(GenSplits.class.getSimpleName());
+
+try {
+ jcommander.parse(args);
+} catch (ParameterException pe) {
+ System.err.println(pe.getMessage());
+ jcommander.usage();
+ System.exit(-1);
+}
+
+if (opts.args == null || opts.args.size() != 1) {
+ jcommander.usage();
System.exit(-1);
}
-int numTablets = Integer.parseInt(args[0]);
+int numTablets = Integer.parseInt(opts.args.get(0));
if (numTablets 1) {
System.err.println(ERROR: numTablets 1);
System.exit(-1);
}
+if (opts.minRow = opts.maxRow) {
+ System.err.println(ERROR: min = max);
+ System.exit(-1);
+}
+
int numSplits = numTablets - 1;
-long distance = (Long.MAX_VALUE / numTablets) + 1;
+long distance = ((opts.maxRow - opts.minRow) / numTablets) + 1;
long split = distance;
for (int i = 0; i numSplits; i++) {
- String s = String.format(%016x, split);
+ String s = String.format(%016x, split + opts.minRow);
while (s.charAt(s.length() - 1) == '0') {
s = s.substring(0, s.length() - 1);
svn commit: r1434696 - in /accumulo/trunk/core/src: main/java/org/apache/accumulo/core/file/rfile/RFile.java test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
Author: kturner
Date: Thu Jan 17 14:45:57 2013
New Revision: 1434696
URL: http://svn.apache.org/viewvc?rev=1434696view=rev
Log:
ACCUMULO-962 fixed issue where rfile would read more data than it was supposed
to. Added unit test that reproduced the bug.
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java?rev=1434696r1=1434695r2=1434696view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
Thu Jan 17 14:45:57 2013
@@ -642,6 +642,7 @@ public class RFile {
private void _seek(Range range) throws IOException {
this.range = range;
+ this.checkRange = true;
if (blockCount == 0) {
// its an empty file
Modified:
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java?rev=1434696r1=1434695r2=1434696view=diff
==
---
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
(original)
+++
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
Thu Jan 17 14:45:57 2013
@@ -28,6 +28,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Random;
@@ -1442,6 +1443,63 @@ public class RFileTest {
trf.closeReader();
}
+ @Test
+ public void testReseekUnconsumed() throws Exception {
+TestRFile trf = new TestRFile();
+
+trf.openWriter();
+
+for (int i = 0; i 2500; i++) {
+ trf.writer.append(nk(nf(r_, i), cf1, cq1, L1, 42), nv(foo +
i));
+}
+
+trf.closeWriter();
+trf.openReader();
+
+SetByteSequence cfs = Collections.emptySet();
+
+Random rand = new Random();
+
+for (int count = 0; count 100; count++) {
+
+ int start = rand.nextInt(2300);
+ Range range = new Range(nk(nf(r_, start), cf1, cq1, L1, 42),
nk(nf(r_, start + 100), cf1, cq1, L1, 42));
+
+ trf.reader.seek(range, cfs, false);
+
+ int numToScan = rand.nextInt(100);
+
+ for (int j = 0; j numToScan; j++) {
+assertTrue(trf.reader.hasTop());
+assertEquals(nk(nf(r_, start + j), cf1, cq1, L1, 42),
trf.reader.getTopKey());
+trf.reader.next();
+ }
+
+ assertTrue(trf.reader.hasTop());
+ assertEquals(nk(nf(r_, start + numToScan), cf1, cq1, L1, 42),
trf.reader.getTopKey());
+
+ // seek a little forward from the last range and read a few keys within
the unconsumed portion of the last range
+
+ int start2 = start + numToScan + rand.nextInt(3);
+ int end2 = start2 + rand.nextInt(3);
+
+ range = new Range(nk(nf(r_, start2), cf1, cq1, L1, 42),
nk(nf(r_, end2), cf1, cq1, L1, 42));
+ trf.reader.seek(range, cfs, false);
+
+ for (int j = start2; j = end2; j++) {
+assertTrue(trf.reader.hasTop());
+assertEquals(nk(nf(r_, j), cf1, cq1, L1, 42),
trf.reader.getTopKey());
+trf.reader.next();
+ }
+
+ assertFalse(trf.reader.hasTop());
+
+}
+
+trf.closeReader();
+ }
+
+
@Test(expected = NullPointerException.class)
public void testMissingUnreleasedVersions() throws Exception {
runVersionTest(5);
svn commit: r1434762 - in /accumulo/trunk/core/src: main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIter
Author: kturner
Date: Thu Jan 17 16:28:48 2013
New Revision: 1434762
URL: http://svn.apache.org/viewvc?rev=1434762view=rev
Log:
ACCUMULO-956 checkin of patch from Brain Loss
Added:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
Added:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java?rev=1434762view=auto
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
(added)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
Thu Jan 17 16:28:48 2013
@@ -0,0 +1,599 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Map;
+
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.OptionDescriber;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.WrappingIterator;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.VisibilityEvaluator;
+import org.apache.accumulo.core.security.VisibilityParseException;
+import org.apache.accumulo.core.util.BadArgumentException;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.collections.map.LRUMap;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+/**
+ * The KeyTransformingIterator allows portions of a key (except for the row)
+ * to be transformed. This iterator handles the details that come with
modifying
+ * keys (i.e., that the sort order could change). In order to do so, however,
+ * the iterator must put all keys sharing the same prefix in memory. Prefix
+ * is defined as the parts of the key that are not modified by this iterator.
+ * That is, if the iterator modifies column qualifier and timestamp, then the
+ * prefix is row and column family. In that case, the iterator must load all
+ * column qualifiers for each row/column family pair into memory. Given this
+ * constraint, care must be taken by users of this iterator to ensure it is
+ * not run in such a way that will overrun memory in a tablet server.
+ * p
+ * If the implementing iterator is transforming column families, then it
+ * must also override {@link #untransformColumnFamilies(Collection)} to handle
+ * the case when column families are fetched at scan time. The fetched column
+ * families will/must be in the transformed space, and the untransformed column
+ * families need to be passed to this iterator's source. If it is not possible
+ * to write a reverse transformation (e.g., the column family transformation
+ * depends on the row value or something like that), then the iterator must
+ * not fetch specific column families (or only fetch column families that are
+ * known to not transform at all).
+ * p
+ * If the implementing iterator is transforming column visibilities, then
+ * users must be careful NOT to fetch column qualifiers from the scanner.
+ * The reason for this is due to ACCUMULO-??? (insert issue number).
+ * p
+ * If the implementing iterator is transforming column visibilities, then the
+ * user should be sure to supply authorizations via the {@link #AUTH_OPT}
+ * iterator option (note
svn commit: r1434881 - in /accumulo/trunk/core/src: main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIter
Author: kturner
Date: Thu Jan 17 20:01:44 2013
New Revision: 1434881
URL: http://svn.apache.org/viewvc?rev=1434881view=rev
Log:
ACCUMULO-956 fixed some issues and added unit test for transforming iterator
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java?rev=1434881r1=1434880r2=1434881view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
Thu Jan 17 20:01:44 2013
@@ -279,26 +279,40 @@ abstract public class KeyTransformingIte
* @return {@code true} if the key is visible or iterator is not scanning,
and {@code false} if not
*/
protected boolean canSee(Key key) {
-// Ensure that the visibility (which could have been transformed) parses.
+// Ensure that the visibility (which could have been transformed) parses.
Must always do this check, even if visibility is not evaluated.
ByteSequence visibility = key.getColumnVisibilityData();
-ColumnVisibility colVis = (ColumnVisibility)
parsedVisibilitiesCache.get(visibility);
-if (colVis == null) {
+ColumnVisibility colVis = null;
+Boolean parsed = (Boolean) parsedVisibilitiesCache.get(visibility);
+if (parsed == null) {
try {
colVis = new ColumnVisibility(visibility.toArray());
+parsedVisibilitiesCache.put(visibility, Boolean.TRUE);
} catch (BadArgumentException e) {
-log.error(Transformation produced an invalid visibility: +
visibility);
-throw e;
+log.error(Parse error after transformation : + visibility);
+parsedVisibilitiesCache.put(visibility, Boolean.FALSE);
+if (scanning) {
+ return false;
+} else {
+ throw e;
+}
}
+} else if (!parsed) {
+ if (scanning)
+return false;
+ else
+throw new IllegalStateException();
}
Boolean visible = canSeeColumnFamily(key);
-if (!scanning || !visible || ve == null || visibleCache == null)
+if (!scanning || !visible || ve == null || visibleCache == null ||
visibility.length() == 0)
return visible;
visible = (Boolean) visibleCache.get(visibility);
if (visible == null) {
try {
+if (colVis == null)
+ colVis = new ColumnVisibility(visibility.toArray());
visible = ve.evaluate(colVis);
visibleCache.put(visibility, visible);
} catch (VisibilityParseException e) {
Modified:
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java?rev=1434881r1=1434880r2=1434881view=diff
==
---
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
(original)
+++
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
Thu Jan 17 20:01:44 2013
@@ -16,18 +16,24 @@
*/
package org.apache.accumulo.core.iterators.user;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.SortedMap;
import java.util.TreeMap;
+import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
@@ -43,9 +49,9 @@ import org.apache.accumulo.core.data.Par
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.WrappingIterator;
-import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope
svn commit: r1434955 - in /accumulo/trunk/core/src: main/java/org/apache/accumulo/core/iterators/user/ test/java/org/apache/accumulo/core/iterators/user/
Author: kturner
Date: Thu Jan 17 23:17:20 2013
New Revision: 1434955
URL: http://svn.apache.org/viewvc?rev=1434955view=rev
Log:
ACCUMULO-956 generlaized transforming iterator, added some sanity checks to it,
added some more unit test, added some static config methods
Added:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
- copied, changed from r1434936,
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java
- copied, changed from r1434936,
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
Removed:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
Copied:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
(from r1434936,
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java)
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java?p2=accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.javap1=accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.javar1=1434936r2=1434955rev=1434955view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
Thu Jan 17 23:17:20 2013
@@ -23,7 +23,10 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Map;
+import java.util.NoSuchElementException;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.PartialKey;
@@ -40,12 +43,13 @@ import org.apache.accumulo.core.security
import org.apache.accumulo.core.security.VisibilityParseException;
import org.apache.accumulo.core.util.BadArgumentException;
import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.collections.BufferOverflowException;
import org.apache.commons.collections.map.LRUMap;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
/**
- * The KeyTransformingIterator allows portions of a key (except for the row)
to be transformed. This iterator handles the details that come with modifying
keys
+ * The TransformingIterator allows portions of a key (except for the row) to
be transformed. This iterator handles the details that come with modifying keys
* (i.e., that the sort order could change). In order to do so, however, the
iterator must put all keys sharing the same prefix in memory. Prefix is defined
as
* the parts of the key that are not modified by this iterator. That is, if
the iterator modifies column qualifier and timestamp, then the prefix is row and
* column family. In that case, the iterator must load all column qualifiers
for each row/column family pair into memory. Given this constraint, care must be
@@ -70,8 +74,11 @@ import org.apache.log4j.Logger;
* major and minor compactions. It should also be noted that this iterator
implements the security filtering rather than relying on a follow-on iterator
to do
* it so that we ensure the test is performed.
*/
-abstract public class KeyTransformingIterator extends WrappingIterator
implements OptionDescriber {
+abstract public class TransformingIterator extends WrappingIterator implements
OptionDescriber {
public static final String AUTH_OPT = authorizations;
+ public static final String MAX_BUFFER_SIZE_OPT = maxBufferSize;
+ private static final long DEFAULT_MAX_BUFFER_SIZE = 1000;
+
protected Logger log = Logger.getLogger(getClass());
protected ArrayListPairKey,Value keys = new ArrayListPairKey,Value();
@@ -84,6 +91,7 @@ abstract public class KeyTransformingIte
private VisibilityEvaluator ve = null;
private LRUMap visibleCache = null;
private LRUMap parsedVisibilitiesCache = null;
+ private long maxBufferSize;
private static ComparatorPairKey,Value keyComparator = new
ComparatorPairKey,Value() {
@Override
@@ -92,7 +100,7 @@ abstract public class KeyTransformingIte
}
};
- public KeyTransformingIterator() {}
+ public TransformingIterator() {}
@Override
public void init(SortedKeyValueIteratorKey,Value source,
MapString,String options
svn commit: r1435371 [3/3] - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/client/admin/ core/src/main/java/org/apache/accumulo/core/client/mock/ core/src/main/java/org/apache/accumu
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java?rev=1435371r1=1435370r2=1435371view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
Fri Jan 18 22:06:44 2013
@@ -104,6 +104,7 @@ import org.apache.accumulo.core.util.she
import org.apache.accumulo.core.util.shell.commands.InfoCommand;
import org.apache.accumulo.core.util.shell.commands.InsertCommand;
import org.apache.accumulo.core.util.shell.commands.InterpreterCommand;
+import org.apache.accumulo.core.util.shell.commands.ListCompactionsCommand;
import org.apache.accumulo.core.util.shell.commands.ListIterCommand;
import org.apache.accumulo.core.util.shell.commands.ListScansCommand;
import org.apache.accumulo.core.util.shell.commands.MaxRowCommand;
@@ -293,7 +294,7 @@ public class Shell extends ShellOptions
Command[] dataCommands = {new DeleteCommand(), new DeleteManyCommand(),
new DeleteRowsCommand(), new EGrepCommand(), new FormatterCommand(),
new InterpreterCommand(), new GrepCommand(), new
ImportDirectoryCommand(), new InsertCommand(), new MaxRowCommand(), new
ScanCommand()};
-Command[] debuggingCommands = {new ClasspathCommand(), new DebugCommand(),
new ListScansCommand(), new TraceCommand()};
+Command[] debuggingCommands = {new ClasspathCommand(), new DebugCommand(),
new ListScansCommand(), new ListCompactionsCommand(), new TraceCommand()};
Command[] execCommands = {new ExecfileCommand(), new HistoryCommand()};
Command[] exitCommands = {new ByeCommand(), new ExitCommand(), new
QuitCommand()};
Command[] helpCommands = {new AboutCommand(), new HelpCommand(), new
InfoCommand(), new QuestionCommand()};
Added:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ActiveCompactionIterator.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ActiveCompactionIterator.java?rev=1435371view=auto
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ActiveCompactionIterator.java
(added)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ActiveCompactionIterator.java
Fri Jan 18 22:06:44 2013
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util.shell.commands;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.admin.ActiveCompaction;
+import org.apache.accumulo.core.client.admin.InstanceOperations;
+import org.apache.accumulo.core.util.Duration;
+
+class ActiveCompactionIterator implements IteratorString {
+
+ private InstanceOperations instanceOps;
+ private IteratorString tsIter;
+ private IteratorString compactionIter;
+
+ private static String maxDecimal(double count) {
+if (count 9.995)
+ return String.format(%.2f, count);
+if (count 99.95)
+ return String.format(%.1f, count);
+return String.format(%.0f, count);
+ }
+
+ private static String shortenCount(long count) {
+if (count 1000)
+ return count + ;
+if (count 100)
+ return maxDecimal(count / 1000.0) + K;
+if (count 10)
+ return maxDecimal(count / 100.0) + M;
+return maxDecimal(count / 10.0) + B;
+ }
+
+ private void readNext() {
+final ListString compactions = new ArrayListString();
+
+while (tsIter.hasNext()) {
+
+ final String tserver = tsIter.next();
+ try {
+ListActiveCompaction acl = instanceOps.getActiveCompactions(tserver);
+
+acl = new
svn commit: r1435383 - /accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
Author: kturner
Date: Fri Jan 18 22:38:20 2013
New Revision: 1435383
URL: http://svn.apache.org/viewvc?rev=1435383view=rev
Log:
ACCUMULO-956 applied patch from Brian Loss
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java?rev=1435383r1=1435382r2=1435383view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
Fri Jan 18 22:38:20 2013
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.NoSuchElementException;
@@ -125,10 +126,16 @@ abstract public class TransformingIterat
@Override
public IteratorOptions describeOptions() {
-String desc = This iterator allows keys to be transformed.;
+String desc = This iterator allows ranges of key to be transformed (with
the exception of row transformations).;
String authDesc = Comma-separated list of user's scan authorizations.
+ If excluded or empty, then no visibility check is performed on
transformed keys.;
-return new IteratorOptions(getClass().getSimpleName(), desc,
Collections.singletonMap(AUTH_OPT, authDesc), null);
+String bufferDesc = Maximum buffer size (in accumulo memory spec) to use
for buffering keys before throwing a BufferOverflowException. +
+ Users should keep this limit in mind when deciding what to
transform. That is, if transforming the column family for example, then all +
+ keys sharing the same row and column family must fit within
this limit (along with their associated values);
+HashMapString,String namedOptions = new HashMapString,String();
+namedOptions.put(AUTH_OPT, authDesc);
+namedOptions.put(MAX_BUFFER_SIZE_OPT, bufferDesc);
+return new IteratorOptions(getClass().getSimpleName(), desc, namedOptions,
null);
}
@Override
@@ -166,6 +173,8 @@ abstract public class TransformingIterat
copy.parsedVisibilitiesCache.putAll(parsedVisibilitiesCache);
}
+copy.maxBufferSize = maxBufferSize;
+
return copy;
}
@@ -223,15 +232,17 @@ abstract public class TransformingIterat
}
}
- private class RangeIterator implements SortedKeyValueIteratorKey,Value {
+ private static class RangeIterator implements
SortedKeyValueIteratorKey,Value {
private SortedKeyValueIteratorKey,Value source;
private Key prefixKey;
+private PartialKey keyPrefix;
private boolean hasTop = false;
-RangeIterator(SortedKeyValueIteratorKey,Value source, Key prefixKey) {
+RangeIterator(SortedKeyValueIteratorKey,Value source, Key prefixKey,
PartialKey keyPrefix) {
this.source = source;
this.prefixKey = prefixKey;
+ this.keyPrefix = keyPrefix;
}
@Override
@@ -242,7 +253,7 @@ abstract public class TransformingIterat
@Override
public boolean hasTop() {
// only have a top if the prefix matches
- return hasTop = source.hasTop() source.getTopKey().equals(prefixKey,
getKeyPrefix());
+ return hasTop = source.hasTop() source.getTopKey().equals(prefixKey,
keyPrefix);
}
@Override
@@ -285,7 +296,7 @@ abstract public class TransformingIterat
keys.clear();
final Key prefixKey = super.hasTop() ? new Key(super.getTopKey()) : null;
-transformRange(new RangeIterator(getSource(), prefixKey), new KVBuffer() {
+transformRange(new RangeIterator(getSource(), prefixKey, getKeyPrefix()),
new KVBuffer() {
long appened = 0;
@@ -608,7 +619,7 @@ abstract public class TransformingIterat
}
/**
- * Transforms {@code originalKey}. This method must not change the row part
of the key, and must only change the parts of the key after the return value of
+ * Transforms {@code input}. This method must not change the row part of the
key, and must only change the parts of the key after the return value of
* {@link #getKeyPrefix()}. Implementors must also remember to copy the
delete flag from {@code originalKey} onto the new key. Or, implementors should
use one
* of the helper methods to produce the new key. See any of the
replaceKeyParts methods.
*
@@ -624,7 +635,6 @@ abstract public class TransformingIterat
* @see #replaceKeyParts(Key, Text, Text)
* @see #replaceKeyParts(Key, Text, Text, Text)
*/
-
abstract protected void transformRange
svn commit: r1435402 - /accumulo/site/trunk/content/people.mdtext
Author: kturner Date: Fri Jan 18 22:50:25 2013 New Revision: 1435402 URL: http://svn.apache.org/viewvc?rev=1435402view=rev Log: CMS commit to accumulo by kturner Modified: accumulo/site/trunk/content/people.mdtext Modified: accumulo/site/trunk/content/people.mdtext URL: http://svn.apache.org/viewvc/accumulo/site/trunk/content/people.mdtext?rev=1435402r1=1435401r2=1435402view=diff == --- accumulo/site/trunk/content/people.mdtext (original) +++ accumulo/site/trunk/content/people.mdtext Fri Jan 18 22:50:25 2013 @@ -66,23 +66,24 @@ Contributors table trthusername/ththname/ththorganization/ththtimezone/th/tr - tr class=eventd/tdtdChris Bennight/tdtd/tdtd/td/tr - tr class=oddtd/tdtdChris McCubbin/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr - tr class=eventd/tdtdChristian Rohling/tdtd/tdtd/td/tr - tr class=oddtd/tdtdDennis Patrone/tdtda href=http://www.jhuapl.edu/The Johns Hopkins UniversitybrApplied Physics Laboratory/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr - tr class=eventd/tdtdEd Kohlwey/tdtda href=http://www.boozallen.com/Booz Allen Hamilton/td/atd/td/tr - tr class=oddtdedwardyoon/tdtdEdward Yoon/tdtd/tdtd/td/tr - tr class=eventd/tdtdJay Shipper/tdtd/tdtd/td/tr - tr class=oddtd/tdtdJesse Yates/tdtd/tdtd/td/tr - tr class=eventd/tdtdJim Klucar/tdtd/tdtd/td/tr - tr class=oddtd/tdtdJoe Skora/tdtd/tdtd/td/tr - tr class=eventd/tdtdLaura Peaslee/tdtd/tdtd/td/tr - tr class=oddtd/tdtdLuke Brassard/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr - tr class=eventd/tdtdMatthew Kirkley/tdtd/tdtd/td/tr - tr class=oddtd/tdtdMichael Wall/tdtd/tdtd/td/tr - tr class=eventd/tdtdPhil Eberhardt/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr - tr class=oddtd/tdtdSapah Shah/tdtd/tdtd/td/tr - tr class=eventd/tdtdScott Kuehn/tdtd/tdtd/td/tr - tr class=oddtd/tdtdTim Halloran/tdtd/tdtd/td/tr - tr class=eventd/tdtdTravis Pinney/tdtd/tdtd/td/tr + tr class=eventd/tdtdBrian Loss/tdtd/tdtd/td/tr + tr class=oddtd/tdtdChris Bennight/tdtd/tdtd/td/tr + tr class=eventd/tdtdChris McCubbin/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr + tr class=oddtd/tdtdChristian Rohling/tdtd/tdtd/td/tr + tr class=eventd/tdtdDennis Patrone/tdtda href=http://www.jhuapl.edu/The Johns Hopkins UniversitybrApplied Physics Laboratory/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr + tr class=oddtd/tdtdEd Kohlwey/tdtda href=http://www.boozallen.com/Booz Allen Hamilton/td/atd/td/tr + tr class=eventdedwardyoon/tdtdEdward Yoon/tdtd/tdtd/td/tr + tr class=oddtd/tdtdJay Shipper/tdtd/tdtd/td/tr + tr class=eventd/tdtdJesse Yates/tdtd/tdtd/td/tr + tr class=oddtd/tdtdJim Klucar/tdtd/tdtd/td/tr + tr class=eventd/tdtdJoe Skora/tdtd/tdtd/td/tr + tr class=oddtd/tdtdLaura Peaslee/tdtd/tdtd/td/tr + tr class=eventd/tdtdLuke Brassard/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr + tr class=oddtd/tdtdMatthew Kirkley/tdtd/tdtd/td/tr + tr class=eventd/tdtdMichael Wall/tdtd/tdtd/td/tr + tr class=oddtd/tdtdPhil Eberhardt/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr + tr class=eventd/tdtdSapah Shah/tdtd/tdtd/td/tr + tr class=oddtd/tdtdScott Kuehn/tdtd/tdtd/td/tr + tr class=eventd/tdtdTim Halloran/tdtd/tdtd/td/tr + tr class=oddtd/tdtdTravis Pinney/tdtd/tdtd/td/tr /table \ No newline at end of file
svn commit: r847231 - in /websites/production/accumulo/content: ./ 1.4/apidocs/
Author: kturner Date: Fri Jan 18 22:50:48 2013 New Revision: 847231 Log: Publishing svnmucc operation to accumulo site by kturner Added: websites/production/accumulo/content/ - copied from r847230, websites/staging/accumulo/trunk/content/ websites/production/accumulo/content/1.4/apidocs/ - copied from r847230, websites/production/accumulo/content/1.4/apidocs/
svn commit: r1435416 - in /accumulo/trunk/core/src/main/java/org/apache/accumulo/core: client/admin/ client/mock/ util/shell/ util/shell/commands/
Author: kturner
Date: Fri Jan 18 23:38:08 2013
New Revision: 1435416
URL: http://svn.apache.org/viewvc?rev=1435416view=rev
Log:
ACCUMULO-513 Added API and shell command for pinging tablet servers.
Added:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/PingCommand.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/PingIterator.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java?rev=1435416r1=1435415r2=1435416view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
Fri Jan 18 23:38:08 2013
@@ -90,6 +90,15 @@ public interface InstanceOperations {
public ListActiveCompaction getActiveCompactions(String tserver) throws
AccumuloException, AccumuloSecurityException;
/**
+ * Throws an exception if a tablet server can not be contacted.
+ *
+ * @param tserver
+ * The tablet server address should be of the form ip
address:port
+ * @throws AccumuloException
+ */
+ public void ping(String tserver) throws AccumuloException;
+
+ /**
* Test to see if the instance can load the given class as the given type.
*
* @param className
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java?rev=1435416r1=1435415r2=1435416view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
Fri Jan 18 23:38:08 2013
@@ -35,11 +35,15 @@ import org.apache.accumulo.core.client.i
import org.apache.accumulo.core.client.impl.thrift.ConfigurationType;
import org.apache.accumulo.core.master.thrift.MasterClientService;
import org.apache.accumulo.core.security.thrift.AuthInfo;
+import org.apache.accumulo.core.security.thrift.ThriftSecurityException;
import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
import org.apache.accumulo.core.util.ArgumentChecker;
import org.apache.accumulo.core.util.ThriftUtil;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
/**
* Provides a class for administering the accumulo instance
@@ -194,4 +198,28 @@ public class InstanceOperationsImpl impl
}
return as;
}
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
org.apache.accumulo.core.client.admin.InstanceOperations#ping(java.lang.String)
+ */
+ @Override
+ public void ping(String tserver) throws AccumuloException {
+Client client = null;
+try {
+ client = ThriftUtil.getTServerClient(tserver,
instance.getConfiguration());
+ client.getTabletServerStatus(Tracer.traceInfo(), credentials);
+} catch (TTransportException e) {
+ throw new AccumuloException(e);
+} catch (ThriftSecurityException e) {
+ throw new AccumuloException(e);
+} catch (TException e) {
+ throw new AccumuloException(e);
+} finally {
+ if (client != null) {
+ThriftUtil.returnClient(client);
+ }
+}
+ }
}
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java?rev=1435416r1=1435415r2=1435416view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
Fri Jan 18 23:38:08 2013
@@ -125,4 +125,15 @@ public class MockInstanceOperations impl
public
svn commit: r1436933 - /accumulo/site/trunk/content/people.mdtext
Author: kturner Date: Tue Jan 22 14:04:53 2013 New Revision: 1436933 URL: http://svn.apache.org/viewvc?rev=1436933view=rev Log: CMS commit to accumulo by kturner Modified: accumulo/site/trunk/content/people.mdtext Modified: accumulo/site/trunk/content/people.mdtext URL: http://svn.apache.org/viewvc/accumulo/site/trunk/content/people.mdtext?rev=1436933r1=1436932r2=1436933view=diff == --- accumulo/site/trunk/content/people.mdtext (original) +++ accumulo/site/trunk/content/people.mdtext Tue Jan 22 14:04:53 2013 @@ -66,7 +66,7 @@ Contributors table trthusername/ththname/ththorganization/ththtimezone/th/tr - tr class=eventd/tdtdBrian Loss/tdtd/tdtd/td/tr + tr class=eventd/tdtdBrian Loss/tdtda href=http://www.praxiseng.com/Praxis Engineering/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr tr class=oddtd/tdtdChris Bennight/tdtd/tdtd/td/tr tr class=eventd/tdtdChris McCubbin/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr tr class=oddtd/tdtdChristian Rohling/tdtd/tdtd/td/tr
svn commit: r1436995 [1/2] - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/client/admin/ core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/ core/src/main/java/org/apach
Author: kturner
Date: Tue Jan 22 15:32:31 2013
New Revision: 1436995
URL: http://svn.apache.org/viewvc?rev=1436995view=rev
Log:
ACCUMULO-875 made list compactions API return files being compacted instead of
a count. Shell still shows a count
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/ActiveCompaction.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/IteratorConfig.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TIteratorSetting.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ActiveCompactionIterator.java
accumulo/trunk/core/src/main/thrift/tabletserver.thrift
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java?rev=1436995r1=1436994r2=1436995view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java
Tue Jan 22 15:32:31 2013
@@ -110,10 +110,10 @@ public class ActiveCompaction {
}
/**
- * @return number of files compaction is reading
+ * @return the files the compaction is reading from
*/
- public int getInputFiles() {
+ public ListString getInputFiles() {
return tac.getInputFiles();
}
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/ActiveCompaction.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/ActiveCompaction.java?rev=1436995r1=1436994r2=1436995view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/ActiveCompaction.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/ActiveCompaction.java
Tue Jan 22 15:32:31 2013
@@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory;
private static final org.apache.thrift.protocol.TField EXTENT_FIELD_DESC =
new org.apache.thrift.protocol.TField(extent,
org.apache.thrift.protocol.TType.STRUCT, (short)1);
private static final org.apache.thrift.protocol.TField AGE_FIELD_DESC = new
org.apache.thrift.protocol.TField(age, org.apache.thrift.protocol.TType.I64,
(short)2);
- private static final org.apache.thrift.protocol.TField
INPUT_FILES_FIELD_DESC = new org.apache.thrift.protocol.TField(inputFiles,
org.apache.thrift.protocol.TType.I32, (short)3);
+ private static final org.apache.thrift.protocol.TField
INPUT_FILES_FIELD_DESC = new org.apache.thrift.protocol.TField(inputFiles,
org.apache.thrift.protocol.TType.LIST, (short)3);
private static final org.apache.thrift.protocol.TField
OUTPUT_FILE_FIELD_DESC = new org.apache.thrift.protocol.TField(outputFile,
org.apache.thrift.protocol.TType.STRING, (short)4);
private static final org.apache.thrift.protocol.TField TYPE_FIELD_DESC = new
org.apache.thrift.protocol.TField(type, org.apache.thrift.protocol.TType.I32,
(short)5);
private static final org.apache.thrift.protocol.TField REASON_FIELD_DESC =
new org.apache.thrift.protocol.TField(reason,
org.apache.thrift.protocol.TType.I32, (short)6);
@@ -69,7 +69,7 @@ import org.slf4j.LoggerFactory;
public org.apache.accumulo.core.data.thrift.TKeyExtent extent; // required
public long age; // required
- public int inputFiles; // required
+ public ListString inputFiles; // required
public String outputFile; // required
/**
*
@@ -185,9 +185,8 @@ import org.slf4j.LoggerFactory;
// isset id assignments
private static final int __AGE_ISSET_ID = 0;
- private static final int __INPUTFILES_ISSET_ID = 1;
- private static final int __ENTRIESREAD_ISSET_ID = 2;
- private static final int __ENTRIESWRITTEN_ISSET_ID = 3;
+ private static final int __ENTRIESREAD_ISSET_ID = 1;
+ private static final int __ENTRIESWRITTEN_ISSET_ID = 2;
private byte __isset_bitfield = 0;
public static final Map_Fields, org.apache.thrift.meta_data.FieldMetaData
metaDataMap;
static {
@@ -197,7 +196,8 @@ import org.slf4j.LoggerFactory;
tmpMap.put(_Fields.AGE, new
org.apache.thrift.meta_data.FieldMetaData(age,
org.apache.thrift.TFieldRequirementType.DEFAULT,
new
org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64
svn commit: r1437054 - /accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
Author: kturner
Date: Tue Jan 22 17:18:10 2013
New Revision: 1437054
URL: http://svn.apache.org/viewvc?rev=1437054view=rev
Log:
ACCUMULO-977 modified listscans and listcompactions to not retry on a bad
tserver
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java?rev=1437054r1=1437053r2=1437054view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
Tue Jan 22 17:18:10 2013
@@ -36,7 +36,6 @@ import org.apache.accumulo.core.client.i
import org.apache.accumulo.core.master.thrift.MasterClientService;
import org.apache.accumulo.core.security.thrift.AuthInfo;
import org.apache.accumulo.core.security.thrift.ThriftSecurityException;
-import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
import org.apache.accumulo.core.util.ArgumentChecker;
import org.apache.accumulo.core.util.ThriftUtil;
@@ -147,22 +146,29 @@ public class InstanceOperationsImpl impl
@Override
public ListActiveScan getActiveScans(String tserver) throws
AccumuloException, AccumuloSecurityException {
-Listorg.apache.accumulo.core.tabletserver.thrift.ActiveScan tas =
ThriftUtil.execute(tserver, instance.getConfiguration(),
-new
ClientExecReturnListorg.apache.accumulo.core.tabletserver.thrift.ActiveScan,TabletClientService.Client()
{
- @Override
- public Listorg.apache.accumulo.core.tabletserver.thrift.ActiveScan
execute(TabletClientService.Client client) throws Exception {
-return client.getActiveScans(Tracer.traceInfo(), credentials);
- }
-});
-ListActiveScan as = new ArrayListActiveScan();
-for (org.apache.accumulo.core.tabletserver.thrift.ActiveScan activeScan :
tas) {
- try {
-as.add(new ActiveScan(instance, activeScan));
- } catch (TableNotFoundException e) {
-throw new AccumuloException(e);
+Client client = null;
+try {
+ client = ThriftUtil.getTServerClient(tserver,
instance.getConfiguration());
+
+ ListActiveScan as = new ArrayListActiveScan();
+ for (org.apache.accumulo.core.tabletserver.thrift.ActiveScan activeScan
: client.getActiveScans(Tracer.traceInfo(), credentials)) {
+try {
+ as.add(new ActiveScan(instance, activeScan));
+} catch (TableNotFoundException e) {
+ throw new AccumuloException(e);
+}
}
+ return as;
+} catch (TTransportException e) {
+ throw new AccumuloException(e);
+} catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+} catch (TException e) {
+ throw new AccumuloException(e);
+} finally {
+ if (client != null)
+ThriftUtil.returnClient(client);
}
-return as;
}
/* (non-Javadoc)
@@ -185,18 +191,25 @@ public class InstanceOperationsImpl impl
*/
@Override
public ListActiveCompaction getActiveCompactions(String tserver) throws
AccumuloException, AccumuloSecurityException {
-Listorg.apache.accumulo.core.tabletserver.thrift.ActiveCompaction tas =
ThriftUtil.execute(tserver, instance.getConfiguration(),
-new
ClientExecReturnListorg.apache.accumulo.core.tabletserver.thrift.ActiveCompaction,TabletClientService.Client()
{
- @Override
- public
Listorg.apache.accumulo.core.tabletserver.thrift.ActiveCompaction
execute(TabletClientService.Client client) throws Exception {
-return client.getActiveCompactions(Tracer.traceInfo(),
credentials);
- }
-});
-ListActiveCompaction as = new ArrayListActiveCompaction();
-for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction
activeCompaction : tas) {
- as.add(new ActiveCompaction(instance, activeCompaction));
+Client client = null;
+try {
+ client = ThriftUtil.getTServerClient(tserver,
instance.getConfiguration());
+
+ ListActiveCompaction as = new ArrayListActiveCompaction();
+ for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction
activeCompaction : client.getActiveCompactions(Tracer.traceInfo(),
credentials)) {
+as.add(new ActiveCompaction(instance, activeCompaction));
+ }
+ return as;
+} catch (TTransportException e) {
+ throw new AccumuloException(e);
+} catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+} catch
svn commit: r1438320 - /accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java
Author: kturner
Date: Fri Jan 25 04:40:47 2013
New Revision: 1438320
URL: http://svn.apache.org/viewvc?rev=1438320view=rev
Log:
ACCUMULO-988 prevent concurrent table compactions with per compaction iterators
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java?rev=1438320r1=1438319r2=1438320view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java
Fri Jan 25 04:40:47 2013
@@ -217,14 +217,13 @@ public class CompactRange extends Master
long flushID = Long.parseLong(new String(tokens[0]));
flushID++;
+ if (tokens.length 1) {
+throw new ThriftTableOperationException(tableId, null,
TableOperation.COMPACT, TableOperationExceptionType.OTHER,
+Another compaction with iterators is running);
+ }
+
String txidString = String.format(%016x, tid);
StringBuilder encodedIterators = new StringBuilder();
- for (int i = 1; i tokens.length; i++) {
-if (tokens[i].startsWith(txidString))
- continue; // skip self
-encodedIterators.append(,);
-encodedIterators.append(tokens[i]);
- }
if (iterators != null iterators.getIterators().size() 0) {
Hex hex = new Hex();
svn commit: r1438668 - in /accumulo/trunk/proxy: ./ src/main/java/org/apache/accumulo/proxy/ src/test/java/org/apache/accumulo/proxy/
Author: kturner
Date: Fri Jan 25 19:51:44 2013
New Revision: 1438668
URL: http://svn.apache.org/viewvc?rev=1438668view=rev
Log:
ACCUMULO-991 committing patch from Jim Klucar
Modified:
accumulo/trunk/proxy/proxy.properties
accumulo/trunk/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/TestProxyInstanceOperations.java
accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/TestProxyReadWrite.java
accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/TestProxySecurityOperations.java
accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/TestProxyTableOperations.java
Modified: accumulo/trunk/proxy/proxy.properties
URL:
http://svn.apache.org/viewvc/accumulo/trunk/proxy/proxy.properties?rev=1438668r1=1438667r2=1438668view=diff
==
--- accumulo/trunk/proxy/proxy.properties (original)
+++ accumulo/trunk/proxy/proxy.properties Fri Jan 25 19:51:44 2013
@@ -2,6 +2,8 @@ accumulo.proxy.apis=org.apache.accumulo.
org.apache.accumulo.proxy.ProxyServer.useMockInstance=false
org.apache.accumulo.proxy.thrift.AccumuloProxy.implementor=org.apache.accumulo.proxy.ProxyServer
+org.apache.accumulo.proxy.thrift.AccumuloProxy.protocolFactory=org.apache.thrift.protocol.TCompactProtocol$Factory
+org.apache.accumulo.proxy.thrift.AccumuloProxy.transportFactory=org.apache.thrift.transport.TFramedTransport$Factory
org.apache.accumulo.proxy.thrift.AccumuloProxy.port=42424
org.apache.accumulo.proxy.ProxyServer.instancename=test
Modified:
accumulo/trunk/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java?rev=1438668r1=1438667r2=1438668view=diff
==
--- accumulo/trunk/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
(original)
+++ accumulo/trunk/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java Fri
Jan 25 19:51:44 2013
@@ -24,11 +24,11 @@ import java.util.Properties;
import org.apache.accumulo.core.cli.Help;
import org.apache.thrift.TProcessor;
-import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.protocol.TProtocolFactory;
import org.apache.thrift.server.THsHaServer;
import org.apache.thrift.server.TServer;
-import org.apache.thrift.transport.TFramedTransport;
import org.apache.thrift.transport.TNonblockingServerSocket;
+import org.apache.thrift.transport.TTransportFactory;
import com.beust.jcommander.IStringConverter;
import com.beust.jcommander.Parameter;
@@ -83,15 +83,20 @@ public class Proxy {
Class? implementor = Class.forName(opts.prop.getProperty(api +
.implementor));
+ Class? extends TProtocolFactory protoFactoryClass =
Class.forName(opts.prop.getProperty(api +
.protocolFactory)).asSubclass(TProtocolFactory.class);
+ Class? extends TTransportFactory transportFactoryClass =
Class.forName(opts.prop.getProperty(api + .transportFactory)).asSubclass(
+ TTransportFactory.class);
+
int port = Integer.parseInt(opts.prop.getProperty(api + .port));
- TServer server = createProxyServer(apiclass, implementor, port,
opts.prop);
+ TServer server = createProxyServer(apiclass, implementor, port,
protoFactoryClass, transportFactoryClass, opts.prop);
server.serve();
}
}
- public static TServer createProxyServer(Class? api, Class? implementor,
final int port, Properties properties) throws Exception {
+ public static TServer createProxyServer(Class? api, Class? implementor,
final int port, Class? extends TProtocolFactory protoClass,
+ Class? extends TTransportFactory transportFactoryClass, Properties
properties) throws Exception {
final TNonblockingServerSocket socket = new TNonblockingServerSocket(port);
-
+
// create the implementor
Object impl =
implementor.getConstructor(Properties.class).newInstance(properties);
@@ -104,8 +109,8 @@ public class Proxy {
THsHaServer.Args args = new THsHaServer.Args(socket);
args.processor(processor);
-args.transportFactory(new TFramedTransport.Factory());
-args.protocolFactory(new TCompactProtocol.Factory());
+args.transportFactory(transportFactoryClass.newInstance());
+args.protocolFactory(protoClass.newInstance());
return new THsHaServer(args);
}
Modified:
accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java?rev=1438668r1=1438667r2=1438668view=diff
==
---
accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy
svn commit: r1438671 - /accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java
Author: kturner
Date: Fri Jan 25 19:54:06 2013
New Revision: 1438671
URL: http://svn.apache.org/viewvc?rev=1438671view=rev
Log:
ACCUMULO-988 make compact Range idempotent again
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java?rev=1438671r1=1438670r2=1438671view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java
Fri Jan 25 19:54:06 2013
@@ -217,12 +217,16 @@ public class CompactRange extends Master
long flushID = Long.parseLong(new String(tokens[0]));
flushID++;
- if (tokens.length 1) {
+ String txidString = String.format(%016x, tid);
+
+ for (int i = 1; i tokens.length; i++) {
+if (tokens[i].startsWith(txidString))
+ continue; // skip self
+
throw new ThriftTableOperationException(tableId, null,
TableOperation.COMPACT, TableOperationExceptionType.OTHER,
Another compaction with iterators is running);
}
- String txidString = String.format(%016x, tid);
StringBuilder encodedIterators = new StringBuilder();
if (iterators != null iterators.getIterators().size() 0) {
svn commit: r1438837 - in /accumulo/trunk/server/src/main/java/org/apache/accumulo/server: master/tableOps/CompactRange.java tabletserver/Tablet.java
Author: kturner
Date: Sat Jan 26 03:54:43 2013
New Revision: 1438837
URL: http://svn.apache.org/viewvc?rev=1438837view=rev
Log:
ACCUMULO-989 ensured that per compaction iterators are not used for tablets
outside of compaction range
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java?rev=1438837r1=1438836r2=1438837view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java
Sat Jan 26 03:54:43 2013
@@ -16,6 +16,10 @@
*/
package org.apache.accumulo.server.master.tableOps;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -36,10 +40,7 @@ import org.apache.accumulo.core.data.Key
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.master.state.tables.TableState;
-import org.apache.accumulo.core.tabletserver.thrift.IteratorConfig;
-import org.apache.accumulo.core.tabletserver.thrift.TIteratorSetting;
import org.apache.accumulo.fate.Repo;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.fate.zookeeper.ZooReaderWriter.Mutator;
@@ -51,6 +52,8 @@ import org.apache.accumulo.server.util.M
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.commons.codec.binary.Hex;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
import org.apache.log4j.Logger;
import org.apache.thrift.TException;
import org.apache.zookeeper.KeeperException.NoNodeException;
@@ -187,21 +190,104 @@ class CompactionDriver extends MasterRep
}
+
public class CompactRange extends MasterRepo {
private static final long serialVersionUID = 1L;
private String tableId;
private byte[] startRow;
private byte[] endRow;
- private IteratorConfig iterators;
+ private byte[] iterators;
+ public static class CompactionIterators implements Writable {
+byte[] startRow;
+byte[] endRow;
+ListIteratorSetting iterators;
+
+public CompactionIterators(byte[] startRow, byte[] endRow,
ListIteratorSetting iterators) {
+ this.startRow = startRow;
+ this.endRow = endRow;
+ this.iterators = iterators;
+}
+
+public CompactionIterators() {
+ startRow = null;
+ endRow = null;
+ iterators = Collections.emptyList();
+}
+
+@Override
+public void write(DataOutput out) throws IOException {
+ out.writeBoolean(startRow != null);
+ if (startRow != null) {
+out.writeInt(startRow.length);
+out.write(startRow);
+ }
+
+ out.writeBoolean(endRow != null);
+ if (endRow != null) {
+out.writeInt(endRow.length);
+out.write(endRow);
+ }
+
+ out.writeInt(iterators.size());
+ for (IteratorSetting is : iterators) {
+is.write(out);
+ }
+}
+
+@Override
+public void readFields(DataInput in) throws IOException {
+ if (in.readBoolean()) {
+startRow = new byte[in.readInt()];
+in.readFully(startRow);
+ } else {
+startRow = null;
+ }
+
+ if (in.readBoolean()) {
+endRow = new byte[in.readInt()];
+in.readFully(endRow);
+ } else {
+endRow = null;
+ }
+
+ int num = in.readInt();
+ iterators = new ArrayListIteratorSetting(num);
+
+ for (int i = 0; i num; i++) {
+iterators.add(new IteratorSetting(in));
+ }
+}
+
+public Text getEndRow() {
+ if (endRow == null)
+return null;
+ return new Text(endRow);
+}
+
+public Text getStartRow() {
+ if (startRow == null)
+return null;
+ return new Text(startRow);
+}
+
+public ListIteratorSetting getIterators() {
+ return iterators;
+}
+ }
+
public CompactRange(String tableId, byte[] startRow, byte[] endRow,
ListIteratorSetting iterators) throws ThriftTableOperationException {
this.tableId = tableId;
this.startRow = startRow.length == 0 ? null : startRow;
this.endRow = endRow.length == 0 ? null : endRow;
-// store as IteratorConfig because its
svn commit: r1439694 - in /accumulo/branches/1.4: ./ src/ src/core/ src/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java src/server/ src/server/src/
Author: kturner
Date: Mon Jan 28 23:33:27 2013
New Revision: 1439694
URL: http://svn.apache.org/viewvc?rev=1439694view=rev
Log:
ACCUMULO-977 made listscans continue on bad tserver (merged from trunk)
Modified:
accumulo/branches/1.4/ (props changed)
accumulo/branches/1.4/src/ (props changed)
accumulo/branches/1.4/src/core/ (props changed)
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
accumulo/branches/1.4/src/server/ (props changed)
accumulo/branches/1.4/src/server/src/ (props changed)
Propchange: accumulo/branches/1.4/
--
Merged /accumulo/trunk:r1437054
Propchange: accumulo/branches/1.4/src/
--
Merged /accumulo/trunk/src:r1437054
Merged /accumulo/trunk:r1437054
Propchange: accumulo/branches/1.4/src/core/
--
Merged /accumulo/trunk/src/core:r1437054
Merged /accumulo/trunk/core:r1437054
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java?rev=1439694r1=1439693r2=1439694view=diff
==
---
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
(original)
+++
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
Mon Jan 28 23:33:27 2013
@@ -34,12 +34,14 @@ import org.apache.accumulo.core.client.i
import org.apache.accumulo.core.client.impl.thrift.ConfigurationType;
import org.apache.accumulo.core.master.thrift.MasterClientService;
import org.apache.accumulo.core.security.thrift.AuthInfo;
-import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.security.thrift.ThriftSecurityException;
import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
import org.apache.accumulo.core.util.ArgumentChecker;
import org.apache.accumulo.core.util.ThriftUtil;
import org.apache.accumulo.core.zookeeper.ZooCache;
import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
/**
* Provides a class for administering the accumulo instance
@@ -143,22 +145,30 @@ public class InstanceOperationsImpl impl
@Override
public ListActiveScan getActiveScans(String tserver) throws
AccumuloException, AccumuloSecurityException {
-Listorg.apache.accumulo.core.tabletserver.thrift.ActiveScan tas =
ThriftUtil.execute(tserver, instance.getConfiguration(),
-new
ClientExecReturnListorg.apache.accumulo.core.tabletserver.thrift.ActiveScan,TabletClientService.Iface()
{
- @Override
- public Listorg.apache.accumulo.core.tabletserver.thrift.ActiveScan
execute(Iface client) throws Exception {
-return client.getActiveScans(null, credentials);
- }
-});
-ListActiveScan as = new ArrayListActiveScan();
-for (org.apache.accumulo.core.tabletserver.thrift.ActiveScan activeScan :
tas) {
- try {
-as.add(new ActiveScan(instance, activeScan));
- } catch (TableNotFoundException e) {
-throw new AccumuloException(e);
+
+Iface client = null;
+try {
+ client = ThriftUtil.getTServerClient(tserver,
instance.getConfiguration());
+
+ ListActiveScan as = new ArrayListActiveScan();
+ for (org.apache.accumulo.core.tabletserver.thrift.ActiveScan activeScan
: client.getActiveScans(null, credentials)) {
+try {
+ as.add(new ActiveScan(instance, activeScan));
+} catch (TableNotFoundException e) {
+ throw new AccumuloException(e);
+}
}
+ return as;
+} catch (TTransportException e) {
+ throw new AccumuloException(e);
+} catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+} catch (TException e) {
+ throw new AccumuloException(e);
+} finally {
+ if (client != null)
+ThriftUtil.returnClient(client);
}
-return as;
}
/* (non-Javadoc)
Propchange: accumulo/branches/1.4/src/server/
--
Merged /accumulo/trunk/src/server:r1437054
Merged /accumulo/trunk/server:r1437054
Propchange: accumulo/branches/1.4/src/server/src/
--
Merged /accumulo/trunk/server/src:r1437054
Merged /accumulo/trunk/src/server/src:r1437054
svn commit: r1440221 - /accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
Author: kturner
Date: Tue Jan 29 23:47:35 2013
New Revision: 1440221
URL: http://svn.apache.org/viewvc?rev=1440221view=rev
Log:
ACCUMULO-976 allow user compactions to queue when system compaction is queued
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java?rev=1440221r1=1440220r2=1440221view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
Tue Jan 29 23:47:35 2013
@@ -31,6 +31,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -47,8 +48,6 @@ import java.util.concurrent.atomic.Atomi
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.accumulo.trace.instrument.Span;
-import org.apache.accumulo.trace.instrument.Trace;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.IteratorSetting;
@@ -124,6 +123,8 @@ import org.apache.accumulo.server.util.M
import org.apache.accumulo.server.util.TabletOperations;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import org.apache.hadoop.conf.Configuration;
@@ -406,7 +407,7 @@ public class Tablet {
private DatafileManager datafileManager;
private volatile boolean majorCompactionInProgress = false;
private volatile boolean majorCompactionWaitingToStart = false;
- private volatile boolean majorCompactionQueued = false;
+ private SetMajorCompactionReason majorCompactionQueued =
Collections.synchronizedSet(EnumSet.noneOf(MajorCompactionReason.class));
private volatile boolean minorCompactionInProgress = false;
private volatile boolean minorCompactionWaitingToStart = false;
@@ -2857,7 +2858,7 @@ public class Tablet {
if (tabletServer.isMajorCompactionDisabled()) {
// this will make compaction task that were queued when shutdown was
// initiated exit
-majorCompactionQueued = false;
+majorCompactionQueued.remove(reason);
return;
}
@@ -2909,12 +2910,12 @@ public class Tablet {
synchronized boolean initiateMajorCompaction(MajorCompactionReason reason) {
-if (closing || closed || !needsMajorCompaction(reason) ||
majorCompactionInProgress || majorCompactionQueued) {
+if (closing || closed || !needsMajorCompaction(reason) ||
majorCompactionInProgress || majorCompactionQueued.contains(reason)) {
return false;
}
-majorCompactionQueued = true;
-
+majorCompactionQueued.add(reason);
+
tabletResources.executeMajorCompaction(getExtent(), new
CompactionRunner(reason));
return false;
@@ -3240,6 +3241,12 @@ public class Tablet {
// compaction was canceled
return majCStats;
}
+
+ synchronized (this) {
+if (lastCompactID = compactionId.getFirst())
+ // already compacted
+ return majCStats;
+ }
}
compactionIterators = compactionId.getSecond();
@@ -3359,18 +3366,19 @@ public class Tablet {
// Always trace majC
Span span = Trace.on(majorCompaction);
-try {
- synchronized (this) {
-// check that compaction is still needed - defer to splitting
-majorCompactionQueued = false;
-
-if (closing || closed || !needsMajorCompaction(reason) ||
majorCompactionInProgress || needsSplit()) {
- return null;
-}
-
-majorCompactionInProgress = true;
+
+synchronized (this) {
+ // check that compaction is still needed - defer to splitting
+ majorCompactionQueued.remove(reason);
+
+ if (closing || closed || !needsMajorCompaction(reason) ||
majorCompactionInProgress || needsSplit()) {
+return null;
}
+ majorCompactionInProgress = true;
+}
+
+try {
majCStats = _majorCompact(reason);
if (reason == MajorCompactionReason.CHOP) {
MetadataTable.chopped(getExtent(), this.tabletServer.getLock());
@@ -3458,7 +3466,7 @@ public class Tablet {
}
public
svn commit: r1440939 - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/util/ proxy/ proxy/src/main/java/org/apache/accumulo/proxy/ proxy/src/test/java/org/apache/accumulo/proxy/
Author: kturner
Date: Thu Jan 31 12:53:23 2013
New Revision: 1440939
URL: http://svn.apache.org/viewvc?rev=1440939view=rev
Log:
ACCUMULO-991 Removed ability to configure transport in proxy. Also made unit
test use different protocols and found/fixed some bugs as a result.
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
accumulo/trunk/proxy/proxy.properties
accumulo/trunk/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
accumulo/trunk/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
accumulo/trunk/proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java
accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/TestProxyInstanceOperations.java
accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/TestProxyReadWrite.java
accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/TestProxySecurityOperations.java
accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/TestProxyTableOperations.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java?rev=1440939r1=1440938r2=1440939view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
Thu Jan 31 12:53:23 2013
@@ -28,7 +28,7 @@ public class ByteBufferUtil {
public static byte[] toBytes(ByteBuffer buffer) {
if (buffer == null)
return null;
-return Arrays.copyOfRange(buffer.array(), buffer.position(),
buffer.remaining());
+return Arrays.copyOfRange(buffer.array(), buffer.position(),
buffer.limit());
}
public static ListByteBuffer toByteBuffers(Collectionbyte[] bytesList) {
Modified: accumulo/trunk/proxy/proxy.properties
URL:
http://svn.apache.org/viewvc/accumulo/trunk/proxy/proxy.properties?rev=1440939r1=1440938r2=1440939view=diff
==
--- accumulo/trunk/proxy/proxy.properties (original)
+++ accumulo/trunk/proxy/proxy.properties Thu Jan 31 12:53:23 2013
@@ -3,7 +3,6 @@ accumulo.proxy.apis=org.apache.accumulo.
org.apache.accumulo.proxy.ProxyServer.useMockInstance=false
org.apache.accumulo.proxy.thrift.AccumuloProxy.implementor=org.apache.accumulo.proxy.ProxyServer
org.apache.accumulo.proxy.thrift.AccumuloProxy.protocolFactory=org.apache.thrift.protocol.TCompactProtocol$Factory
-org.apache.accumulo.proxy.thrift.AccumuloProxy.transportFactory=org.apache.thrift.transport.TFramedTransport$Factory
org.apache.accumulo.proxy.thrift.AccumuloProxy.port=42424
org.apache.accumulo.proxy.ProxyServer.instancename=test
Modified:
accumulo/trunk/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java?rev=1440939r1=1440938r2=1440939view=diff
==
--- accumulo/trunk/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
(original)
+++ accumulo/trunk/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java Thu
Jan 31 12:53:23 2013
@@ -27,8 +27,8 @@ import org.apache.thrift.TProcessor;
import org.apache.thrift.protocol.TProtocolFactory;
import org.apache.thrift.server.THsHaServer;
import org.apache.thrift.server.TServer;
+import org.apache.thrift.transport.TFramedTransport;
import org.apache.thrift.transport.TNonblockingServerSocket;
-import org.apache.thrift.transport.TTransportFactory;
import com.beust.jcommander.IStringConverter;
import com.beust.jcommander.Parameter;
@@ -84,17 +84,15 @@ public class Proxy {
Class? implementor = Class.forName(opts.prop.getProperty(api +
.implementor));
Class? extends TProtocolFactory protoFactoryClass =
Class.forName(opts.prop.getProperty(api +
.protocolFactory)).asSubclass(TProtocolFactory.class);
- Class? extends TTransportFactory transportFactoryClass =
Class.forName(opts.prop.getProperty(api + .transportFactory)).asSubclass(
- TTransportFactory.class);
int port = Integer.parseInt(opts.prop.getProperty(api + .port));
- TServer server = createProxyServer(apiclass, implementor, port,
protoFactoryClass, transportFactoryClass, opts.prop);
+ TServer server = createProxyServer(apiclass, implementor, port,
protoFactoryClass, opts.prop);
server.serve();
}
}
public static TServer createProxyServer(Class? api, Class? implementor,
final int port, Class? extends TProtocolFactory protoClass,
- Class? extends TTransportFactory transportFactoryClass, Properties
svn commit: r1441504 - in /accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf: DefaultConfiguration.java Property.java
Author: kturner
Date: Fri Feb 1 16:14:00 2013
New Revision: 1441504
URL: http://svn.apache.org/viewvc?rev=1441504view=rev
Log:
ACCUMULO-958 marked incomplete WAL encrpytion feature as experimental so it
will not show up in docs
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java?rev=1441504r1=1441503r2=1441504view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
Fri Feb 1 16:14:00 2013
@@ -47,7 +47,7 @@ public class DefaultConfiguration extend
public IteratorEntryString,String iterator() {
TreeMapString,String entries = new TreeMapString,String();
for (Property prop : Property.values())
- if (!prop.getType().equals(PropertyType.PREFIX))
+ if (!prop.isExprimental() !prop.getType().equals(PropertyType.PREFIX))
entries.put(prop.getKey(), prop.getDefaultValue());
return entries.entrySet().iterator();
@@ -78,6 +78,9 @@ public class DefaultConfiguration extend
ArrayListProperty prefixes = new ArrayListProperty();
TreeMapString,Property sortedProps = new TreeMapString,Property();
for (Property prop : Property.values()) {
+ if (prop.isExprimental())
+continue;
+
if (prop.getType().equals(PropertyType.PREFIX))
prefixes.add(prop);
else
@@ -87,6 +90,9 @@ public class DefaultConfiguration extend
doc.println( pJump to: );
String delimiter = ;
for (Property prefix : prefixes) {
+ if (prefix.isExprimental())
+continue;
+
doc.print(delimiter + a href='# + prefix.name() + ' +
prefix.getKey() + */a);
delimiter = nbsp;|nbsp;;
}
@@ -94,6 +100,10 @@ public class DefaultConfiguration extend
doc.println( table);
for (Property prefix : prefixes) {
+
+ if (prefix.isExprimental())
+continue;
+
doc.println( trtd colspan='5'a name=' + prefix.name() + '
class='large' + prefix.getKey() + */a/td/tr);
doc.println( trtd colspan='5'i + prefix.getDescription() +
/i/td/tr);
if (!prefix.equals(Property.TABLE_CONSTRAINT_PREFIX)
!prefix.equals(Property.TABLE_ITERATOR_PREFIX)
@@ -102,6 +112,9 @@ public class DefaultConfiguration extend
boolean highlight = true;
for (Property prop : sortedProps.values()) {
+if (prop.isExprimental())
+ continue;
+
if (prop.getKey().startsWith(prefix.getKey())) {
doc.println( tr + (highlight ? class='highlight' : ) +
);
highlight = !highlight;
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1441504r1=1441503r2=1441504view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
Fri Feb 1 16:14:00 2013
@@ -29,27 +29,30 @@ import org.apache.accumulo.start.classlo
public enum Property {
// Crypto-related properties
- CRYPTO_PREFIX(crypto., null, PropertyType.PREFIX, Properties in this
category related to the configuration of both default and custom crypto
modules.),
+ CRYPTO_PREFIX(crypto., null, PropertyType.PREFIX, Properties in this
category related to the configuration of both default and custom crypto
modules.,
+ true),
CRYPTO_MODULE_CLASS(
crypto.module.class,
NullCryptoModule,
PropertyType.STRING,
- Fully qualified class name of the class that implements the
CryptoModule interface, to be used in setting up encryption at rest for the WAL
and (future) other parts of the code.),
- CRYPTO_CIPHER_SUITE(crypto.cipher.suite, NullCipher,
PropertyType.STRING, Describes the cipher suite to use for the write-ahead
log),
+ Fully qualified class name of the class that implements the
CryptoModule interface, to be used in setting up encryption at rest for the WAL
and (future) other parts of the code.,
+ true),
+ CRYPTO_CIPHER_SUITE(crypto.cipher.suite, NullCipher,
PropertyType.STRING, Describes the cipher suite to use for the write-ahead
log, true),
CRYPTO_CIPHER_ALGORITHM_NAME(crypto.cipher.algorithm.name, NullCipher,
PropertyType.STRING,
- States the name of the algorithm used in the corresponding cipher
suite. Do
svn commit: r1441559 - in /accumulo/trunk/core/src/main/java/org/apache/accumulo/core: conf/DefaultConfiguration.java conf/Property.java util/shell/commands/ConfigCommand.java
Author: kturner
Date: Fri Feb 1 18:02:20 2013
New Revision: 1441559
URL: http://svn.apache.org/viewvc?rev=1441559view=rev
Log:
ACCUMULO-958 fixed spelling, hid experimental options in shell
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ConfigCommand.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java?rev=1441559r1=1441558r2=1441559view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
Fri Feb 1 18:02:20 2013
@@ -47,7 +47,7 @@ public class DefaultConfiguration extend
public IteratorEntryString,String iterator() {
TreeMapString,String entries = new TreeMapString,String();
for (Property prop : Property.values())
- if (!prop.isExprimental() !prop.getType().equals(PropertyType.PREFIX))
+ if (!prop.isExperimental()
!prop.getType().equals(PropertyType.PREFIX))
entries.put(prop.getKey(), prop.getDefaultValue());
return entries.entrySet().iterator();
@@ -78,7 +78,7 @@ public class DefaultConfiguration extend
ArrayListProperty prefixes = new ArrayListProperty();
TreeMapString,Property sortedProps = new TreeMapString,Property();
for (Property prop : Property.values()) {
- if (prop.isExprimental())
+ if (prop.isExperimental())
continue;
if (prop.getType().equals(PropertyType.PREFIX))
@@ -90,7 +90,7 @@ public class DefaultConfiguration extend
doc.println( pJump to: );
String delimiter = ;
for (Property prefix : prefixes) {
- if (prefix.isExprimental())
+ if (prefix.isExperimental())
continue;
doc.print(delimiter + a href='# + prefix.name() + ' +
prefix.getKey() + */a);
@@ -101,7 +101,7 @@ public class DefaultConfiguration extend
doc.println( table);
for (Property prefix : prefixes) {
- if (prefix.isExprimental())
+ if (prefix.isExperimental())
continue;
doc.println( trtd colspan='5'a name=' + prefix.name() + '
class='large' + prefix.getKey() + */a/td/tr);
@@ -112,7 +112,7 @@ public class DefaultConfiguration extend
boolean highlight = true;
for (Property prop : sortedProps.values()) {
-if (prop.isExprimental())
+if (prop.isExperimental())
continue;
if (prop.getKey().startsWith(prefix.getKey())) {
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1441559r1=1441558r2=1441559view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
Fri Feb 1 18:02:20 2013
@@ -373,7 +373,7 @@ public enum Property {
return this.description;
}
- public boolean isExprimental() {
+ public boolean isExperimental() {
return experimental;
}
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ConfigCommand.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ConfigCommand.java?rev=1441559r1=1441558r2=1441559view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ConfigCommand.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ConfigCommand.java
Fri Feb 1 18:02:20 2013
@@ -53,7 +53,7 @@ public class ConfigCommand extends Comma
final Token cmd = new Token(getName());
final Token sub = new Token(- + setOpt.getOpt());
for (Property p : Property.values()) {
- if (!(p.getKey().endsWith(.))) {
+ if (!(p.getKey().endsWith(.)) !p.isExperimental()) {
sub.addSubcommand(new Token(p.toString()));
}
}
svn commit: r1443085 - in /accumulo/trunk/test/src/test/java/org/apache/accumulo: fate/zookeeper/ZooLockTest.java test/MiniAccumuloClusterTest.java
Author: kturner
Date: Wed Feb 6 17:38:45 2013
New Revision: 1443085
URL: http://svn.apache.org/viewvc?rev=1443085view=rev
Log:
ACCUMULO-954 suppressed warnings from zookeeper during unit test
Modified:
accumulo/trunk/test/src/test/java/org/apache/accumulo/fate/zookeeper/ZooLockTest.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java
Modified:
accumulo/trunk/test/src/test/java/org/apache/accumulo/fate/zookeeper/ZooLockTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/fate/zookeeper/ZooLockTest.java?rev=1443085r1=1443084r2=1443085view=diff
==
---
accumulo/trunk/test/src/test/java/org/apache/accumulo/fate/zookeeper/ZooLockTest.java
(original)
+++
accumulo/trunk/test/src/test/java/org/apache/accumulo/fate/zookeeper/ZooLockTest.java
Wed Feb 6 17:38:45 2013
@@ -83,7 +83,7 @@ public class ZooLockTest {
folder.create();
-Logger.getLogger(org.apache.zookeeper).setLevel(Level.WARN);
+Logger.getLogger(org.apache.zookeeper).setLevel(Level.ERROR);
accumulo = new MiniAccumuloCluster(folder.getRoot(), superSecret);
Modified:
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java?rev=1443085r1=1443084r2=1443085view=diff
==
---
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java
(original)
+++
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java
Wed Feb 6 17:38:45 2013
@@ -57,7 +57,7 @@ public class MiniAccumuloClusterTest {
folder.create();
-Logger.getLogger(org.apache.zookeeper).setLevel(Level.WARN);
+Logger.getLogger(org.apache.zookeeper).setLevel(Level.ERROR);
accumulo = new MiniAccumuloCluster(folder.getRoot(), superSecret);
svn commit: r1443790 - in /accumulo/trunk: fate/src/main/java/org/apache/accumulo/fate/zookeeper/ server/src/main/java/org/apache/accumulo/server/gc/ server/src/main/java/org/apache/accumulo/server/ma
Author: kturner
Date: Thu Feb 7 23:42:57 2013
New Revision: 1443790
URL: http://svn.apache.org/r1443790
Log:
ACCUMULO-954 made zoolock report when its no longer able to monitor lock node
and there does not know the status of the lock
Modified:
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/TServerLockWatcher.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/zookeeper/ZooLock.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/fate/zookeeper/ZooLockTest.java
Modified:
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java?rev=1443790r1=1443789r2=1443790view=diff
==
---
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
(original)
+++
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
Thu Feb 7 23:42:57 2013
@@ -38,11 +38,18 @@ public class ZooLock implements Watcher
public static final String LOCK_PREFIX = zlock-;
public enum LockLossReason {
-LOCK_DELETED, SESSION_EXPIRED, UNKNOWN
+LOCK_DELETED, SESSION_EXPIRED
}
public interface LockWatcher {
void lostLock(LockLossReason reason);
+
+/**
+ * lost the ability to monitor the lock node, and its status is unknown
+ *
+ * @param e
+ */
+void unableToMonitorLockNode(Throwable e);
}
public interface AsyncLockWatcher extends LockWatcher {
@@ -98,6 +105,11 @@ public class ZooLock implements Watcher
lw.lostLock(reason);
}
+@Override
+public void unableToMonitorLockNode(Throwable e) {
+ lw.unableToMonitorLockNode(e);
+}
+
}
public synchronized boolean tryLock(LockWatcher lw, byte data[]) throws
KeeperException, InterruptedException {
@@ -133,8 +145,9 @@ public class ZooLock implements Watcher
Collections.sort(children);
if (children.get(0).equals(myLock)) {
- if (!watchingParent)
-throw new RuntimeException(Can not acquire lock, no longer watching
parent);
+ if (!watchingParent) {
+throw new IllegalStateException(Can not acquire lock, no longer
watching parent : + path);
+ }
this.lockWatcher = lw;
this.lock = myLock;
asyncLock = null;
@@ -222,8 +235,8 @@ public class ZooLock implements Watcher
lostLock(LockLossReason.LOCK_DELETED);
} else if (asyncLock != null event.getType() ==
EventType.NodeDeleted event.getPath().equals(path + / + asyncLock)) {
failedToAcquireLock();
-} else if(event.getState() != KeeperState.Expired) {
- log.warn(Unexpected event watching lock node +event+
+asyncLockPath);
+} else if (event.getState() != KeeperState.Expired (lock !=
null || asyncLock != null)) {
+ log.debug(Unexpected event watching lock node +event+
+asyncLockPath);
try {
Stat stat2 = zooKeeper.getStatus(asyncLockPath, this);
if(stat2 == null){
@@ -232,20 +245,9 @@ public class ZooLock implements Watcher
else if(asyncLock != null)
failedToAcquireLock();
}
- } catch (Exception e) {
+ } catch (Throwable e) {
+lockWatcher.unableToMonitorLockNode(e);
log.error(Failed to stat lock node + asyncLockPath, e);
-
-try {
- // not sure what happened... try to clean lock node up
- zooKeeper.delete(asyncLockPath, -1);
-} catch (Throwable e2) {
- log.debug(Failed to clean up lock node + asyncLockPath,
e2);
-}
-
-if(lock != null)
- lostLock(LockLossReason.UNKNOWN);
-else if(asyncLock != null)
- failedToAcquireLock();
}
}
@@ -348,10 +350,9 @@ public class ZooLock implements Watcher
zooKeeper.getStatus(path, this);
watchingParent = true;
} catch (Exception ex) {
-log.warn(Error resetting watch on ZooLock, ex);
-
-if (lock != null) {
- lostLock
svn commit: r1444259 - /accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
Author: kturner
Date: Fri Feb 8 22:11:22 2013
New Revision: 1444259
URL: http://svn.apache.org/r1444259
Log:
ACCUMULO-1049 modified master to stop locking tserver nodes and just monitor
tserver nodes in zookeeper
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java?rev=1444259r1=1444258r2=1444259view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
Fri Feb 8 22:11:22 2013
@@ -22,13 +22,9 @@ import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
-import org.apache.accumulo.trace.instrument.Tracer;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
@@ -49,6 +45,7 @@ import org.apache.accumulo.server.util.t
import org.apache.accumulo.server.zookeeper.ZooCache;
import org.apache.accumulo.server.zookeeper.ZooLock;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.trace.instrument.Tracer;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
import org.apache.thrift.TException;
@@ -57,7 +54,6 @@ import org.apache.zookeeper.KeeperExcept
import org.apache.zookeeper.KeeperException.NotEmptyException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.Watcher.Event.EventType;
import org.apache.zookeeper.data.Stat;
public class LiveTServerSet implements Watcher {
@@ -189,26 +185,17 @@ public class LiveTServerSet implements W
}
static class TServerInfo {
-ZooLock lock;
TServerConnection connection;
TServerInstance instance;
-TServerLockWatcher watcher;
-TServerInfo(ZooLock lock, TServerInstance instance, TServerConnection
connection, TServerLockWatcher watcher) {
- this.lock = lock;
+TServerInfo(TServerInstance instance, TServerConnection connection) {
this.connection = connection;
this.instance = instance;
- this.watcher = watcher;
-}
-
-void cleanup() throws InterruptedException, KeeperException {
- lock.tryToCancelAsyncLockOrUnlock();
}
};
// Map from tserver master service to server information
private MapString,TServerInfo current = new HashMapString,TServerInfo();
- private HashMapString,Long serversToDelete = new HashMapString,Long();
public LiveTServerSet(Instance instance, AccumuloConfiguration conf,
Listener cback) {
this.cback = cback;
@@ -230,7 +217,7 @@ public class LiveTServerSet implements W
public void run() {
scanServers();
}
-}, 0, 1000);
+}, 0, 5000);
}
public synchronized void scanServers() {
@@ -240,21 +227,11 @@ public class LiveTServerSet implements W
final String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
- IteratorEntryString,Long serversToDelIter =
serversToDelete.entrySet().iterator();
- while (serversToDelIter.hasNext()) {
-EntryString,Long entry = serversToDelIter.next();
-if (System.currentTimeMillis() - entry.getValue() 1) {
- String serverNode = path + / + entry.getKey();
- serversToDelIter.remove();
- deleteServerNode(serverNode);
-}
- }
-
- for (String server : getZooCache().getChildren(path)) {
-if (serversToDelete.containsKey(server))
- continue;
-
-checkServer(updates, doomed, path, server, 2);
+ HashSetString all = new HashSetString(current.keySet());
+ all.addAll(getZooCache().getChildren(path));
+
+ for (String server : all) {
+checkServer(updates, doomed, path, server);
}
// log.debug(Current: + current.keySet());
@@ -275,94 +252,71 @@ public class LiveTServerSet implements W
}
}
- private synchronized void checkServer(final SetTServerInstance updates,
final SetTServerInstance doomed, final String path, final String server,
- int recurse)
- throws TException,
- InterruptedException, KeeperException {
-
-if (recurse == 0)
- return;
+ private synchronized void checkServer(final SetTServerInstance updates,
final SetTServerInstance doomed, final String path, final String server)
+ throws TException, InterruptedException, KeeperException {
-// See if we have an async lock
svn commit: r1444313 - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/Constants.java server/src/main/java/org/apache/accumulo/server/master/Master.java test/system/upgrade_test.sh
Author: kturner
Date: Sat Feb 9 02:10:15 2013
New Revision: 1444313
URL: http://svn.apache.org/r1444313
Log:
ACCUMULO-635 ACCUMULO-1010 got basic upgrade and upgrade test script working...
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
accumulo/trunk/test/system/upgrade_test.sh
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java?rev=1444313r1=1444312r2=1444313view=diff
==
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java
Sat Feb 9 02:10:15 2013
@@ -35,8 +35,8 @@ public class Constants {
// versions should never be negative
public static final Integer WIRE_VERSION = 2;
- public static final int DATA_VERSION = 4;
- public static final int PREV_DATA_VERSION = 3;
+ public static final int DATA_VERSION = 5;
+ public static final int PREV_DATA_VERSION = 4;
// Zookeeper locations
public static final String ZROOT = /accumulo;
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1444313r1=1444312r2=1444313view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
Sat Feb 9 02:10:15 2013
@@ -98,7 +98,6 @@ import org.apache.accumulo.fate.zookeepe
import org.apache.accumulo.server.Accumulo;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.iterators.MetadataBulkLoadFilter;
import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer;
import org.apache.accumulo.server.master.balancer.TabletBalancer;
@@ -157,7 +156,6 @@ import org.apache.accumulo.server.zookee
import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.accumulo.trace.instrument.thrift.TraceWrap;
import org.apache.accumulo.trace.thrift.TInfo;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
@@ -171,8 +169,6 @@ import org.apache.zookeeper.KeeperExcept
import org.apache.zookeeper.KeeperException.NoNodeException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Stat;
/**
@@ -279,53 +275,21 @@ public class Master implements LiveTServ
log.info(Upgrading zookeeper);
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
-
-TablePropUtil.setTableProperty(Constants.METADATA_TABLE_ID,
Property.TABLE_ITERATOR_PREFIX.getKey() + majc.bulkLoadFilter, 20,
-+ MetadataBulkLoadFilter.class.getName());
-
-zoo.putPersistentData(ZooUtil.getRoot(instance) +
Constants.ZTABLE_LOCKS, new byte[0], NodeExistsPolicy.SKIP);
-zoo.putPersistentData(ZooUtil.getRoot(instance) +
Constants.ZHDFS_RESERVATIONS, new byte[0], NodeExistsPolicy.SKIP);
-zoo.putPersistentData(ZooUtil.getRoot(instance) +
Constants.ZNEXT_FILE, new byte[] {'0'}, NodeExistsPolicy.SKIP);
-
-String[] tablePropsToDelete = new String[] {table.scan.cache.size,
table.scan.cache.enable};
-
+
+zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZRECOVERY,
new byte[] {'0'}, NodeExistsPolicy.SKIP);
+
for (String id : Tables.getIdToNameMap(instance).keySet()) {
- zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES
+ / + id + Constants.ZTABLE_FLUSH_ID, 0.getBytes(), NodeExistsPolicy.SKIP);
- zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES
+ / + id + Constants.ZTABLE_COMPACT_ID, 0.getBytes(),
NodeExistsPolicy.SKIP);
+
zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES
+ / + id + Constants.ZTABLE_COMPACT_CANCEL_ID, 0.getBytes(),
NodeExistsPolicy.SKIP);
-
- for (String prop : tablePropsToDelete) {
-String propPath = ZooUtil.getRoot(instance) + Constants.ZTABLES +
/ + id + Constants.ZTABLE_CONF + / + prop;
-if (zoo.exists(propPath))
- zoo.delete(propPath, -1
svn commit: r1445221 - /accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java
Author: kturner
Date: Tue Feb 12 16:02:56 2013
New Revision: 1445221
URL: http://svn.apache.org/r1445221
Log:
ACCUMULO-745 updated example to use new continuous ingest options
Modified:
accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java
Modified:
accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java?rev=1445221r1=1445220r2=1445221view=diff
==
---
accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java
(original)
+++
accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java
Tue Feb 12 16:02:56 2013
@@ -19,9 +19,9 @@ package org.apache.accumulo.instamo;
import java.io.File;
import java.util.UUID;
+import org.apache.accumulo.test.MiniAccumuloCluster;
import org.apache.accumulo.test.continuous.ContinuousIngest;
import org.apache.accumulo.test.continuous.ContinuousVerify;
-import org.apache.accumulo.test.MiniAccumuloCluster;
import org.apache.commons.io.FileUtils;
/**
@@ -32,15 +32,15 @@ public class MapReduceExample {
public static void run(String instanceName, String zookeepers, String
rootPassword, String args[]) throws Exception {
// run continuous ingest to create data. This is not a map reduce job
-ContinuousIngest.main(new String[] {instanceName, zookeepers, root,
rootPassword, ci, 500, 0, Long.MAX_VALUE + , 1000, 1000,
100,
-6, 2, false});
+ContinuousIngest.main(new String[] {-i, instanceName, -z, zookeepers,
-u, root, -p, rootPassword, --table, ci, --num, 500,
+--batchMemory, 100});
String outputDir = FileUtils.getTempDirectoryPath() + File.separator +
ci_verify + UUID.randomUUID().toString();
try {
// run verify map reduce job locally. This jobs looks for holes in the
linked list create by continuous ingest
- ContinuousVerify.main(new String[] {-D, mapred.job.tracker=local,
-D, fs.default.name=file:///, instanceName, zookeepers, root,
rootPassword,
- ci, outputDir, 4, 1, false});
+ ContinuousVerify.main(new String[] {-D, mapred.job.tracker=local,
-D, fs.default.name=file:///, -i, instanceName, -z, zookeepers, -u,
+ root, -p, rootPassword, --table, ci, --output, outputDir,
--maxMappers, 4, --reducers, 1});
} finally {
// delete output dir of mapreduce job
FileUtils.deleteQuietly(new File(outputDir));
svn commit: r1445300 - /accumulo/site/trunk/content/people.mdtext
Author: kturner Date: Tue Feb 12 18:31:56 2013 New Revision: 1445300 URL: http://svn.apache.org/r1445300 Log: CMS commit to accumulo by kturner Modified: accumulo/site/trunk/content/people.mdtext Modified: accumulo/site/trunk/content/people.mdtext URL: http://svn.apache.org/viewvc/accumulo/site/trunk/content/people.mdtext?rev=1445300r1=1445299r2=1445300view=diff == --- accumulo/site/trunk/content/people.mdtext (original) +++ accumulo/site/trunk/content/people.mdtext Tue Feb 12 18:31:56 2013 @@ -70,21 +70,22 @@ Contributors tr class=oddtd/tdtdChris Bennight/tdtd/tdtd/td/tr tr class=eventd/tdtdChris McCubbin/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr tr class=oddtd/tdtdChristian Rohling/tdtd/tdtd/td/tr - tr class=eventd/tdtdDennis Patrone/tdtda href=http://www.jhuapl.edu/The Johns Hopkins UniversitybrApplied Physics Laboratory/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr - tr class=oddtd/tdtdEd Kohlwey/tdtda href=http://www.boozallen.com/Booz Allen Hamilton/a/tdtd/td/tr - tr class=eventdedwardyoon/tdtdEdward Yoon/tdtd/tdtd/td/tr - tr class=oddtd/tdtdJay Shipper/tdtd/tdtd/td/tr - tr class=eventd/tdtdJesse Yates/tdtd/tdtd/td/tr - tr class=oddtd/tdtdJim Klucar/tdtda href=http://www.praxiseng.com/Praxis Engineering/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr - tr class=eventd/tdtdJoe Skora/tdtd/tdtd/td/tr - tr class=oddtd/tdtdLaura Peaslee/tdtd/tdtd/td/tr - tr class=eventd/tdtdLuke Brassard/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr - tr class=oddtd/tdtdMatthew Kirkley/tdtd/tdtd/td/tr - tr class=eventd/tdtdMichael Wall/tdtd/tdtd/td/tr - tr class=oddtd/tdtdOren Falkowitz/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr - tr class=eventd/tdtdPhil Eberhardt/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr - tr class=oddtd/tdtdSapah Shah/tdtd/tdtd/td/tr - tr class=eventd/tdtdScott Kuehn/tdtd/tdtd/td/tr - tr class=oddtd/tdtdTim Halloran/tdtd/tdtd/td/tr - tr class=eventd/tdtdTravis Pinney/tdtd/tdtd/td/tr + tr class=eventd/tdtdDamon Brown/tdtd/tdtd/td/tr + tr class=oddtd/tdtdDennis Patrone/tdtda href=http://www.jhuapl.edu/The Johns Hopkins UniversitybrApplied Physics Laboratory/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr + tr class=eventd/tdtdEd Kohlwey/tdtda href=http://www.boozallen.com/Booz Allen Hamilton/a/tdtd/td/tr + tr class=oddtdedwardyoon/tdtdEdward Yoon/tdtd/tdtd/td/tr + tr class=eventd/tdtdJay Shipper/tdtd/tdtd/td/tr + tr class=oddtd/tdtdJesse Yates/tdtd/tdtd/td/tr + tr class=eventd/tdtdJim Klucar/tdtda href=http://www.praxiseng.com/Praxis Engineering/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr + tr class=oddtd/tdtdJoe Skora/tdtd/tdtd/td/tr + tr class=eventd/tdtdLaura Peaslee/tdtd/tdtd/td/tr + tr class=oddtd/tdtdLuke Brassard/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr + tr class=eventd/tdtdMatthew Kirkley/tdtd/tdtd/td/tr + tr class=oddtd/tdtdMichael Wall/tdtd/tdtd/td/tr + tr class=eventd/tdtdOren Falkowitz/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr + tr class=oddtd/tdtdPhil Eberhardt/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr + tr class=eventd/tdtdSapah Shah/tdtd/tdtd/td/tr + tr class=oddtd/tdtdScott Kuehn
svn commit: r1445303 - /accumulo/site/trunk/content/people.mdtext
Author: kturner Date: Tue Feb 12 18:36:23 2013 New Revision: 1445303 URL: http://svn.apache.org/r1445303 Log: CMS commit to accumulo by kturner Modified: accumulo/site/trunk/content/people.mdtext Modified: accumulo/site/trunk/content/people.mdtext URL: http://svn.apache.org/viewvc/accumulo/site/trunk/content/people.mdtext?rev=1445303r1=1445302r2=1445303view=diff == --- accumulo/site/trunk/content/people.mdtext (original) +++ accumulo/site/trunk/content/people.mdtext Tue Feb 12 18:36:23 2013 @@ -70,7 +70,7 @@ Contributors tr class=oddtd/tdtdChris Bennight/tdtd/tdtd/td/tr tr class=eventd/tdtdChris McCubbin/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr tr class=oddtd/tdtdChristian Rohling/tdtd/tdtd/td/tr - tr class=eventd/tdtdDamon Brown/tdtd/tdtd/td/tr + tr class=eventd/tdtdDamon Brown/tdtda href=http://www.tetraconcepts.com;Tetra Concepts LLC/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr tr class=oddtd/tdtdDennis Patrone/tdtda href=http://www.jhuapl.edu/The Johns Hopkins UniversitybrApplied Physics Laboratory/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr tr class=eventd/tdtdEd Kohlwey/tdtda href=http://www.boozallen.com/Booz Allen Hamilton/a/tdtd/td/tr tr class=oddtdedwardyoon/tdtdEdward Yoon/tdtd/tdtd/td/tr
svn commit: r850390 - in /websites/production/accumulo/content: ./ 1.4/apidocs/
Author: kturner Date: Tue Feb 12 18:37:54 2013 New Revision: 850390 Log: Publishing svnmucc operation to accumulo site by kturner Added: websites/production/accumulo/content/ - copied from r850389, websites/staging/accumulo/trunk/content/ websites/production/accumulo/content/1.4/apidocs/ - copied from r850389, websites/production/accumulo/content/1.4/apidocs/
svn commit: r1445320 - /accumulo/trunk/bin/bootstrap_hdfs.sh
Author: kturner
Date: Tue Feb 12 19:17:27 2013
New Revision: 1445320
URL: http://svn.apache.org/r1445320
Log:
ACCUMULO-645 fixed bootstrap_hdfs.sh
Modified:
accumulo/trunk/bin/bootstrap_hdfs.sh
Modified: accumulo/trunk/bin/bootstrap_hdfs.sh
URL:
http://svn.apache.org/viewvc/accumulo/trunk/bin/bootstrap_hdfs.sh?rev=1445320r1=1445319r2=1445320view=diff
==
--- accumulo/trunk/bin/bootstrap_hdfs.sh (original)
+++ accumulo/trunk/bin/bootstrap_hdfs.sh Tue Feb 12 19:17:27 2013
@@ -61,16 +61,17 @@ fi
#
# Copy all jars in lib to the system context directory
#
-$HADOOP_PREFIX/bin/hadoop fs -moveFromLocal $ACCUMULO_HOME/lib/*.jar
$SYSTEM_CONTEXT_HDFS_DIR /dev/null
+$HADOOP_PREFIX/bin/hadoop fs -moveFromLocal $ACCUMULO_HOME/lib/*.jar
$SYSTEM_CONTEXT_HDFS_DIR /dev/null
$HADOOP_PREFIX/bin/hadoop fs -setrep -R $REP $SYSTEM_CONTEXT_HDFS_DIR
/dev/null
+
#
# We need two of the jars in lib, copy them back out and remove them from the
system context dir
#
$HADOOP_PREFIX/bin/hadoop fs -copyToLocal
$SYSTEM_CONTEXT_HDFS_DIR/log4j-1.2.16.jar $ACCUMULO_HOME/lib/. /dev/null
$HADOOP_PREFIX/bin/hadoop fs -rmr
$SYSTEM_CONTEXT_HDFS_DIR/log4j-1.2.16.jar /dev/null
-$HADOOP_PREFIX/bin/hadoop fs -copyToLocal
$SYSTEM_CONTEXT_HDFS_DIR/commons-vfs2-2.0.jar $ACCUMULO_HOME/lib/.
/dev/null
+$HADOOP_PREFIX/bin/hadoop fs -copyToLocal
$SYSTEM_CONTEXT_HDFS_DIR/commons-vfs2-2.0.jar $ACCUMULO_HOME/lib/.
/dev/null
$HADOOP_PREFIX/bin/hadoop fs -rmr
$SYSTEM_CONTEXT_HDFS_DIR/commons-vfs2-2.0.jar /dev/null
$HADOOP_PREFIX/bin/hadoop fs -copyToLocal
$SYSTEM_CONTEXT_HDFS_DIR/accumulo-start-${ACCUMULO_VERSION}.jar
$ACCUMULO_HOME/lib/. /dev/null
$HADOOP_PREFIX/bin/hadoop fs -rmr
$SYSTEM_CONTEXT_HDFS_DIR/accumulo-start-${ACCUMULO_VERSION}.jar /dev/null
svn commit: r1446318 - /accumulo/site/trunk/content/people.mdtext
Author: kturner Date: Thu Feb 14 20:28:50 2013 New Revision: 1446318 URL: http://svn.apache.org/r1446318 Log: CMS commit to accumulo by kturner Modified: accumulo/site/trunk/content/people.mdtext Modified: accumulo/site/trunk/content/people.mdtext URL: http://svn.apache.org/viewvc/accumulo/site/trunk/content/people.mdtext?rev=1446318r1=1446317r2=1446318view=diff == --- accumulo/site/trunk/content/people.mdtext (original) +++ accumulo/site/trunk/content/people.mdtext Thu Feb 14 20:28:50 2013 @@ -78,14 +78,15 @@ Contributors tr class=oddtd/tdtdJesse Yates/tdtd/tdtd/td/tr tr class=eventd/tdtdJim Klucar/tdtda href=http://www.praxiseng.com/Praxis Engineering/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr tr class=oddtd/tdtdJoe Skora/tdtd/tdtd/td/tr - tr class=eventd/tdtdLaura Peaslee/tdtd/tdtd/td/tr - tr class=oddtd/tdtdLuke Brassard/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr - tr class=eventd/tdtdMatthew Kirkley/tdtd/tdtd/td/tr - tr class=oddtd/tdtdMichael Wall/tdtd/tdtd/td/tr - tr class=eventd/tdtdOren Falkowitz/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr - tr class=oddtd/tdtdPhil Eberhardt/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr - tr class=eventd/tdtdSapah Shah/tdtd/tdtd/td/tr - tr class=oddtd/tdtdScott Kuehn/tdtd/tdtd/td/tr - tr class=eventd/tdtdTim Halloran/tdtd/tdtd/td/tr - tr class=oddtd/tdtdTravis Pinney/tdtd/tdtd/td/tr + tr class=eventd/tdtdKevin Faro/tdtda href=http://www.tetraconcepts.com;Tetra Concepts LLC/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr + tr class=oddtd/tdtdLaura Peaslee/tdtd/tdtd/td/tr + tr class=eventd/tdtdLuke Brassard/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr + tr class=oddtd/tdtdMatthew Kirkley/tdtd/tdtd/td/tr + tr class=eventd/tdtdMichael Wall/tdtd/tdtd/td/tr + tr class=oddtd/tdtdOren Falkowitz/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr + tr class=eventd/tdtdPhil Eberhardt/tdtda href=http://sqrrl.com/sqrrl/a/tdtdET (a href=http://www.timeanddate.com/library/abbreviations/timezones/na/est.html-5/a / a href=http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html-4/a)/td/tr + tr class=oddtd/tdtdSapah Shah/tdtd/tdtd/td/tr + tr class=eventd/tdtdScott Kuehn/tdtd/tdtd/td/tr + tr class=oddtd/tdtdTim Halloran/tdtd/tdtd/td/tr + tr class=eventd/tdtdTravis Pinney/tdtd/tdtd/td/tr /table \ No newline at end of file
svn commit: r850635 - in /websites/production/accumulo/content: ./ 1.4/apidocs/
Author: kturner Date: Thu Feb 14 20:30:09 2013 New Revision: 850635 Log: Publishing svnmucc operation to accumulo site by kturner Added: websites/production/accumulo/content/ - copied from r850634, websites/staging/accumulo/trunk/content/ websites/production/accumulo/content/1.4/apidocs/ - copied from r850634, websites/production/accumulo/content/1.4/apidocs/
svn commit: r1447769 - /accumulo/branches/1.5/
Author: kturner Date: Tue Feb 19 15:36:16 2013 New Revision: 1447769 URL: http://svn.apache.org/r1447769 Log: #ACCUMULO-970 created 1.5 branch Added: accumulo/branches/1.5/ (props changed) - copied from r1447767, accumulo/trunk/ Propchange: accumulo/branches/1.5/ -- --- svn:ignore (added) +++ svn:ignore Tue Feb 19 15:36:16 2013 @@ -0,0 +1,8 @@ +logs +walogs +target +.idea +.project +.settings +.pydevproject +accumulo.iml Propchange: accumulo/branches/1.5/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Tue Feb 19 15:36:16 2013 @@ -0,0 +1,18 @@ +/accumulo/branches/1.3:1309369,1328076,1330246,1330264,1330944,1349971,1354669 +/accumulo/branches/1.3/src:1309369,1328076,1330246,1349971,1354669 +/accumulo/branches/1.4:1305403-1382577,1382613,1388120,1388629,1393868,1396065,1396572,1396616,1396758,1396772,1397048,1397113,1397117,1397176,1397189,1397383,1397700,1397921,1398286,1398308,1398359,1398393,1398399,1398438,1398514,1398801,1399211,1399717,1400976,1402571,1402682,1407301,1423032,1423629,1427864 +/accumulo/branches/1.4/src:1305403-1356900,1358206,1363430,1364778,1365213,1382566,1382923,1388120,1396772,1397048,1397113,1397117,1397176,1397189,1397383,1397700,1397921,1398286,1398308,1398359,1398393,1398399,1398438,1399211,1400976,1402571,1402682,1407157,1423032,1423624,1427919,1428054 +/accumulo/branches/1.4.2:1399210,1402681 +/accumulo/branches/1.4.2/src:1399210,1402681 +/accumulo/branches/ACCUMULO-259:1343822-1438343 +/accumulo/branches/ACCUMULO-672:1357826,1357829,1357842,1357858,1358236,1359163 +/accumulo/trunk/src:1329425,1332224,1332278,1332347,1333047,1333070,1341000,1342373,1350779,1351691,1356400,1359721 +/incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043,1236873,1245632 +/incubator/accumulo/branches/1.3/src:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043,1236873,1245632 +/incubator/accumulo/branches/1.3.5rc:1209938 +/incubator/accumulo/branches/1.3.5rc/src:1209938 +/incubator/accumulo/branches/1.4:1201902-1305402 +/incubator/accumulo/branches/1.4/src:1201899-1305402 +/incubator/accumulo/branches/1.4.0rc:1304025,1305326 +/incubator/accumulo/branches/1.4.0rc/src:1304025,1305326 +/incubator/accumulo/trunk/src:1178656-1201898,1205476,1205570,1208726,1222413,1222719,1222725,1222733-1222734,1296160-1296495
svn commit: r1448472 - in /accumulo/trunk/core/src: main/java/org/apache/accumulo/core/security/Authorizations.java test/java/org/apache/accumulo/core/security/AuthorizationsTest.java
Author: kturner
Date: Wed Feb 20 23:39:52 2013
New Revision: 1448472
URL: http://svn.apache.org/r1448472
Log:
ACCUMULO-1051 checking in patch from Kevin Faro
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/security/AuthorizationsTest.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java?rev=1448472r1=1448471r2=1448472view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
Wed Feb 20 23:39:52 2013
@@ -26,6 +26,8 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
import org.apache.accumulo.core.data.ArrayByteSequence;
import org.apache.accumulo.core.data.ByteSequence;
@@ -37,7 +39,7 @@ public class Authorizations implements I
private static final long serialVersionUID = 1L;
- private HashSetByteSequence auths = new HashSetByteSequence();
+ private SetByteSequence auths = new HashSetByteSequence();
private Listbyte[] authsList = new ArrayListbyte[]();
private Listbyte[] immutableList = Collections.unmodifiableList(authsList);
@@ -74,8 +76,9 @@ public class Authorizations implements I
}
private void checkAuths() {
+SetByteSequence sortedAuths = new TreeSetByteSequence(auths);
-for (ByteSequence bs : auths) {
+for (ByteSequence bs : sortedAuths) {
if (bs.length() == 0) {
throw new IllegalArgumentException(Empty authorization);
}
@@ -233,10 +236,10 @@ public class Authorizations implements I
public String serialize() {
StringBuilder sb = new StringBuilder(HEADER);
String sep = ;
-for (ByteSequence auth : auths) {
+for (byte[] auth : immutableList) {
sb.append(sep);
sep = ,;
- sb.append(new String(Base64.encodeBase64(auth.toArray(;
+ sb.append(new String(Base64.encodeBase64(auth)));
}
return sb.toString();
Modified:
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/security/AuthorizationsTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/security/AuthorizationsTest.java?rev=1448472r1=1448471r2=1448472view=diff
==
---
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/security/AuthorizationsTest.java
(original)
+++
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/security/AuthorizationsTest.java
Wed Feb 20 23:39:52 2013
@@ -43,4 +43,12 @@ public class AuthorizationsTest {
assertEquals(a, b);
}
+ @Test
+ public void testSerialization() {
+Authorizations a1 = new Authorizations(a, b);
+Authorizations a2 = new Authorizations(b, a);
+
+assertEquals(a1, a2);
+assertEquals(a1.serialize(), a2.serialize());
+ }
}
svn commit: r1448692 - in /accumulo/branches/1.5/core/src: main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java test/java/org/apache/accumulo/core/iterators/user/TransformingIte
Author: kturner
Date: Thu Feb 21 15:55:22 2013
New Revision: 1448692
URL: http://svn.apache.org/r1448692
Log:
ACCUMULO-1001 implemented verifyopts for TransformingIterator
Modified:
accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
accumulo/branches/1.5/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java
Modified:
accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java?rev=1448692r1=1448691r2=1448692view=diff
==
---
accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
(original)
+++
accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
Thu Feb 21 15:55:22 2013
@@ -140,9 +140,22 @@ abstract public class TransformingIterat
@Override
public boolean validateOptions(MapString,String options) {
+
+for (String opt : options.keySet()) {
+ try {
+if (options.equals(AUTH_OPT)) {
+ new Authorizations(options.get(opt).getBytes());
+} else if (opt.equals(MAX_BUFFER_SIZE_OPT)) {
+ AccumuloConfiguration.getMemoryInBytes(options.get(opt));
+}
+ } catch (Exception e) {
+throw new IllegalArgumentException(Failed to parse opt + opt +
+ options.get(opt), e);
+ }
+}
+
return true;
}
-
+
@Override
public SortedKeyValueIteratorKey,Value deepCopy(IteratorEnvironment env) {
TransformingIterator copy;
Modified:
accumulo/branches/1.5/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.5/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java?rev=1448692r1=1448691r2=1448692view=diff
==
---
accumulo/branches/1.5/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java
(original)
+++
accumulo/branches/1.5/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java
Thu Feb 21 15:55:22 2013
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -55,6 +56,7 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.hadoop.io.Text;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -388,6 +390,28 @@ public class TransformingIteratorTest {
assertEquals(81, count);
}
+ @Test
+ public void testValidateOptions() {
+TransformingIterator ti = new ColFamReversingKeyTransformingIterator();
+IteratorSetting is = new IteratorSetting(100, cfrkt,
ColFamReversingKeyTransformingIterator.class);
+TransformingIterator.setAuthorizations(is, new Authorizations(A, B));
+TransformingIterator.setMaxBufferSize(is, 1000);
+Assert.assertTrue(ti.validateOptions(is.getOptions()));
+
+MapString,String opts = new HashMapString,String();
+
+opts.put(TransformingIterator.MAX_BUFFER_SIZE_OPT, 10M);
+Assert.assertTrue(ti.validateOptions(is.getOptions()));
+
+opts.clear();
+opts.put(TransformingIterator.MAX_BUFFER_SIZE_OPT, A,B);
+try {
+ ti.validateOptions(opts);
+ Assert.assertFalse(true);
+} catch (IllegalArgumentException e) {}
+
+ }
+
private Key createDeleteKey(String row, String colFam, String colQual,
String colVis, long timestamp) {
Key key = new Key(row, colFam, colQual, colVis, timestamp);
key.setDeleted(true);
svn commit: r1448698 - in /accumulo/trunk: ./ assemble/ core/ core/src/main/java/org/apache/accumulo/core/iterators/user/ core/src/test/java/org/apache/accumulo/core/iterators/user/ examples/ fate/src
Author: kturner
Date: Thu Feb 21 16:01:37 2013
New Revision: 1448698
URL: http://svn.apache.org/r1448698
Log:
ACCUMULO-1001 implemented verifyopts for TransformingIterator (merged from 1.5)
Modified:
accumulo/trunk/ (props changed)
accumulo/trunk/assemble/ (props changed)
accumulo/trunk/core/ (props changed)
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java
accumulo/trunk/examples/ (props changed)
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
(props changed)
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
(props changed)
accumulo/trunk/server/ (props changed)
accumulo/trunk/src/ (props changed)
Propchange: accumulo/trunk/
--
Merged /accumulo/branches/1.5:r1448388-1448697
Propchange: accumulo/trunk/assemble/
--
Merged /accumulo/branches/1.5/assemble:r1448388-1448697
Propchange: accumulo/trunk/core/
--
Merged /accumulo/branches/1.5/core:r1448388-1448697
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java?rev=1448698r1=1448697r2=1448698view=diff
==
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
Thu Feb 21 16:01:37 2013
@@ -140,9 +140,22 @@ abstract public class TransformingIterat
@Override
public boolean validateOptions(MapString,String options) {
+
+for (String opt : options.keySet()) {
+ try {
+if (options.equals(AUTH_OPT)) {
+ new Authorizations(options.get(opt).getBytes());
+} else if (opt.equals(MAX_BUFFER_SIZE_OPT)) {
+ AccumuloConfiguration.getMemoryInBytes(options.get(opt));
+}
+ } catch (Exception e) {
+throw new IllegalArgumentException(Failed to parse opt + opt +
+ options.get(opt), e);
+ }
+}
+
return true;
}
-
+
@Override
public SortedKeyValueIteratorKey,Value deepCopy(IteratorEnvironment env) {
TransformingIterator copy;
Modified:
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java?rev=1448698r1=1448697r2=1448698view=diff
==
---
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java
(original)
+++
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java
Thu Feb 21 16:01:37 2013
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -55,6 +56,7 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.hadoop.io.Text;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -388,6 +390,28 @@ public class TransformingIteratorTest {
assertEquals(81, count);
}
+ @Test
+ public void testValidateOptions() {
+TransformingIterator ti = new ColFamReversingKeyTransformingIterator();
+IteratorSetting is = new IteratorSetting(100, cfrkt,
ColFamReversingKeyTransformingIterator.class);
+TransformingIterator.setAuthorizations(is, new Authorizations(A, B));
+TransformingIterator.setMaxBufferSize(is, 1000);
+Assert.assertTrue(ti.validateOptions(is.getOptions()));
+
+MapString,String opts = new HashMapString,String();
+
+opts.put(TransformingIterator.MAX_BUFFER_SIZE_OPT, 10M);
+Assert.assertTrue(ti.validateOptions(is.getOptions()));
+
+opts.clear();
+opts.put(TransformingIterator.MAX_BUFFER_SIZE_OPT, A,B);
+try {
+ ti.validateOptions(opts);
+ Assert.assertFalse(true);
+} catch (IllegalArgumentException e) {}
+
+ }
+
private Key createDeleteKey(String row, String colFam, String colQual,
String colVis, long timestamp) {
Key key = new Key(row, colFam, colQual, colVis
svn commit: r1448803 - /accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
Author: kturner
Date: Thu Feb 21 21:06:00 2013
New Revision: 1448803
URL: http://svn.apache.org/r1448803
Log:
ACCUMULO-1080 make it easy to see when a offline tablet has walogs
Modified:
accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
Modified:
accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java?rev=1448803r1=1448802r2=1448803view=diff
==
---
accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
(original)
+++
accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
Thu Feb 21 21:06:00 2013
@@ -19,12 +19,12 @@ package org.apache.accumulo.server.util;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.server.master.LiveTServerSet;
import org.apache.accumulo.server.master.LiveTServerSet.Listener;
import org.apache.accumulo.server.master.state.MetaDataTableScanner;
@@ -62,7 +62,7 @@ public class FindOfflineTablets {
TabletState state = locationState.getState(tservers.getCurrentServers());
if (state != TabletState.HOSTED
TableManager.getInstance().getTableState(locationState.extent.getTableId().toString())
!= TableState.OFFLINE)
if (!locationState.extent.equals(Constants.ROOT_TABLET_EXTENT))
- System.out.println(locationState + is + state);
+ System.out.println(locationState + is + state + #walogs: +
locationState.walogs.size());
}
}
svn commit: r1448807 - in /accumulo/trunk: ./ assemble/ core/ examples/ fate/src/main/java/org/apache/accumulo/fate/ fate/src/main/java/org/apache/accumulo/fate/zookeeper/ server/ server/src/main/java
Author: kturner
Date: Thu Feb 21 21:17:55 2013
New Revision: 1448807
URL: http://svn.apache.org/r1448807
Log:
ACCUMULO-1080 make it easy to see when a offline tablet has walogs
ACCUMULO-1081 Add a dependency to commons-math for the configuration randomwalk
tests
Modified:
accumulo/trunk/ (props changed)
accumulo/trunk/assemble/ (props changed)
accumulo/trunk/core/ (props changed)
accumulo/trunk/examples/ (props changed)
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
(props changed)
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
(props changed)
accumulo/trunk/pom.xml
accumulo/trunk/server/ (props changed)
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
accumulo/trunk/src/ (props changed)
accumulo/trunk/test/pom.xml
Propchange: accumulo/trunk/
--
Merged /accumulo/branches/1.5:r1448698-1448805
Propchange: accumulo/trunk/assemble/
--
Merged /accumulo/branches/1.5/assemble:r1448698-1448805
Propchange: accumulo/trunk/core/
--
Merged /accumulo/branches/1.5/core:r1448698-1448805
Propchange: accumulo/trunk/examples/
--
Merged /accumulo/branches/1.5/examples:r1448698-1448805
Propchange:
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
--
Merged
/accumulo/branches/1.5/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java:r1448698-1448805
Propchange:
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
--
Merged
/accumulo/branches/1.5/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java:r1448698-1448805
Modified: accumulo/trunk/pom.xml
URL:
http://svn.apache.org/viewvc/accumulo/trunk/pom.xml?rev=1448807r1=1448806r2=1448807view=diff
==
--- accumulo/trunk/pom.xml (original)
+++ accumulo/trunk/pom.xml Thu Feb 21 21:17:55 2013
@@ -716,6 +716,11 @@
version1.0/version
/dependency
dependency
+groupIdorg.apache.commons/groupId
+artifactIdcommons-math/artifactId
+version2.2/version
+ /dependency
+ dependency
groupIdcom.beust/groupId
artifactIdjcommander/artifactId
version1.30/version
Propchange: accumulo/trunk/server/
--
Merged /accumulo/branches/1.5/server:r1448698-1448805
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java?rev=1448807r1=1448806r2=1448807view=diff
==
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
Thu Feb 21 21:17:55 2013
@@ -19,12 +19,12 @@ package org.apache.accumulo.server.util;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.server.master.LiveTServerSet;
import org.apache.accumulo.server.master.LiveTServerSet.Listener;
import org.apache.accumulo.server.master.state.MetaDataTableScanner;
@@ -62,7 +62,7 @@ public class FindOfflineTablets {
TabletState state = locationState.getState(tservers.getCurrentServers());
if (state != TabletState.HOSTED
TableManager.getInstance().getTableState(locationState.extent.getTableId().toString())
!= TableState.OFFLINE)
if (!locationState.extent.equals(Constants.ROOT_TABLET_EXTENT))
- System.out.println(locationState + is + state);
+ System.out.println(locationState + is + state + #walogs: +
locationState.walogs.size());
}
}
Propchange: accumulo/trunk/src/
--
Merged /accumulo/branches/1.5/src:r1448698-1448805
Modified: accumulo/trunk/test/pom.xml
URL:
http
