Merge branch '1.6.0-SNAPSHOT' Conflicts: core/src/main/java/org/apache/accumulo/core/data/Condition.java core/src/main/java/org/apache/accumulo/core/data/Key.java core/src/main/java/org/apache/accumulo/core/data/Range.java test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/46b0f986 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/46b0f986 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/46b0f986 Branch: refs/heads/master Commit: 46b0f986d7fee32c9b98ea72a97e239944ab7202 Parents: 8856c1f 716ea0e Author: Christopher Tubbs <ctubb...@apache.org> Authored: Wed Apr 9 13:56:59 2014 -0400 Committer: Christopher Tubbs <ctubb...@apache.org> Committed: Wed Apr 9 13:56:59 2014 -0400 ---------------------------------------------------------------------- .../core/client/ClientSideIteratorScanner.java | 2 - .../accumulo/core/client/ConditionalWriter.java | 2 - .../core/client/ConditionalWriterConfig.java | 2 - .../apache/accumulo/core/client/Connector.java | 2 - .../accumulo/core/client/IteratorSetting.java | 4 - .../accumulo/core/client/RowIterator.java | 4 - .../accumulo/core/client/ScannerBase.java | 1 - .../core/client/admin/ActiveCompaction.java | 1 - .../core/client/admin/InstanceOperations.java | 12 -- .../core/client/admin/TableOperations.java | 30 ---- .../core/client/admin/TableOperationsImpl.java | 4 - .../core/client/impl/ThriftTransportPool.java | 16 +- .../core/client/mapred/AbstractInputFormat.java | 2 - .../client/mapred/AccumuloOutputFormat.java | 2 - .../client/mapreduce/AbstractInputFormat.java | 3 +- .../client/mapreduce/AccumuloOutputFormat.java | 2 - .../core/client/mapreduce/InputTableConfig.java | 3 - .../mapreduce/lib/util/ConfiguratorBase.java | 2 - .../core/client/mock/MockBatchDeleter.java | 4 - .../java/org/apache/accumulo/core/data/Key.java | 4 + .../org/apache/accumulo/core/data/Range.java | 6 +- .../accumulo/core/file/rfile/BlockIndex.java | 5 - .../accumulo/core/file/rfile/bcfile/BCFile.java | 15 +- .../core/file/rfile/bcfile/ByteArray.java | 2 - .../accumulo/core/file/rfile/bcfile/Utils.java | 11 -- .../core/iterators/TypedValueCombiner.java | 6 - .../core/iterators/ValueFormatException.java | 6 - .../core/iterators/system/MapFileIterator.java | 8 +- .../core/iterators/user/GrepIterator.java | 3 - .../iterators/user/IntersectingIterator.java | 10 -- .../accumulo/core/iterators/user/RowFilter.java | 1 - .../iterators/user/TransformingIterator.java | 164 +++++++++---------- .../core/iterators/user/VersioningIterator.java | 3 - .../accumulo/core/security/SecurityUtil.java | 1 - .../security/crypto/CryptoModuleFactory.java | 1 - .../security/crypto/CryptoModuleParameters.java | 6 - .../core/client/impl/ScannerOptionsTest.java | 2 - .../client/lexicoder/ReverseLexicoderTest.java | 2 - .../client/mapred/AccumuloInputFormatTest.java | 4 - .../mapreduce/AccumuloInputFormatTest.java | 4 - .../core/client/mock/MockNamespacesTest.java | 8 - .../core/security/VisibilityConstraintTest.java | 3 - .../simple/client/RandomBatchScanner.java | 5 - .../simple/client/RandomBatchWriter.java | 4 - .../simple/client/SequentialBatchWriter.java | 5 - .../simple/client/TraceDumpExample.java | 9 +- .../examples/simple/dirlist/QueryUtil.java | 3 - .../examples/simple/mapreduce/NGramIngest.java | 3 - .../examples/simple/mapreduce/TableToFile.java | 1 - .../accumulo/examples/simple/shard/Query.java | 3 - .../minicluster/MiniAccumuloRunner.java | 2 - .../accumulo/proxy/TestProxyReadWrite.java | 10 -- .../accumulo/server/conf/ConfigSanityCheck.java | 3 - .../server/master/balancer/TabletBalancer.java | 2 - .../server/master/state/TabletStateStore.java | 7 - .../server/metrics/AbstractMetricsImpl.java | 4 - .../server/security/handler/ZKAuthorizor.java | 4 - .../server/security/handler/ZKPermHandler.java | 4 - .../accumulo/server/util/LoginProperties.java | 3 - .../accumulo/server/util/RestoreZookeeper.java | 4 - .../accumulo/server/util/TableDiskUsage.java | 3 - .../accumulo/server/util/TabletServerLocks.java | 3 - .../org/apache/accumulo/tserver/MemValue.java | 4 +- .../apache/accumulo/tserver/TabletServer.java | 14 +- .../tserver/compaction/CompactionStrategy.java | 4 - .../accumulo/tserver/logger/LogReader.java | 1 - .../start/classloader/AccumuloClassLoader.java | 3 - .../start/classloader/vfs/ContextManager.java | 2 - .../vfs/PostDelegatingVFSClassLoader.java | 7 +- .../vfs/providers/HdfsFileSystem.java | 5 - .../accumulo/test/NativeMapPerformanceTest.java | 3 - .../accumulo/test/NativeMapStressTest.java | 3 - .../test/continuous/ContinuousMoru.java | 1 - .../test/continuous/ContinuousVerify.java | 1 - .../test/functional/CacheTestClean.java | 3 - .../accumulo/test/randomwalk/Environment.java | 6 +- .../accumulo/test/randomwalk/Framework.java | 3 - .../apache/accumulo/test/randomwalk/Node.java | 1 - .../randomwalk/concurrent/CheckBalance.java | 5 +- 79 files changed, 125 insertions(+), 396 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriterConfig.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/client/ConditionalWriterConfig.java index 064b9be,a220e62..7671c35 --- a/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriterConfig.java +++ b/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriterConfig.java @@@ -42,11 -42,9 +42,9 @@@ public class ConditionalWriterConfig * * <p> * Any condition that is not visible with this set of authorizations will fail. - * - * @param auths */ public ConditionalWriterConfig setAuthorizations(Authorizations auths) { - ArgumentChecker.notNull(auths); + checkArgument(auths != null, "auths is null"); this.auths = auths; return this; } http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java index ca50ce2,e69f3dd..b966a4a --- a/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java +++ b/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java @@@ -85,11 -85,9 +85,9 @@@ public class IteratorSetting implement /** * Set the iterator's name. Must be a simple alphanumeric identifier. - * - * @param name */ public void setName(String name) { - ArgumentChecker.notNull(name); + checkArgument(name != null, "name is null"); this.name = name; } @@@ -105,11 -103,9 +103,9 @@@ /** * Set the name of the class that implements the iterator. The class does not have to be present on the client, but it must be available to all tablet * servers. - * - * @param iteratorClass */ public void setIteratorClass(String iteratorClass) { - ArgumentChecker.notNull(iteratorClass); + checkArgument(iteratorClass != null, "iteratorClass is null"); this.iteratorClass = iteratorClass; } http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/data/Key.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/data/Key.java index 41754b5,2b44359..bfeb095 --- a/core/src/main/java/org/apache/accumulo/core/data/Key.java +++ b/core/src/main/java/org/apache/accumulo/core/data/Key.java @@@ -715,12 -638,13 +717,13 @@@ public class Key implements WritableCom } /** - * Compare all elements of a key. The elements (row, column family, column qualifier, column visibility, timestamp, and delete marker) are compared in order - * until an unequal element is found. If the row is equal, then compare the column family, etc. The row, column family, column qualifier, and column - * visibility are compared lexographically and sorted ascending. The timestamps are compared numerically and sorted descending so that the most recent data - * comes first. Lastly, a delete marker of true sorts before a delete marker of false. + * Compares this key with another. + * + * @param other key to compare to + * @return comparison result + * @see #compareTo(Key, PartialKey) */ - + @Override public int compareTo(Key other) { return compareTo(other, PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/data/Range.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/data/Range.java index a9469a8,412ee53..56e823e --- a/core/src/main/java/org/apache/accumulo/core/data/Range.java +++ b/core/src/main/java/org/apache/accumulo/core/data/Range.java @@@ -314,12 -310,10 +314,13 @@@ public class Range implements WritableC } /** - * Compares this range to another range. Compares in the order start key, inclusiveness of start key, end key, inclusiveness of end key. Infinite keys sort + * Compares this range to another range. Compares in order: start key, inclusiveness of start key, end key, inclusiveness of end key. Infinite keys sort * first, and non-infinite keys are compared with {@link Key#compareTo(Key)}. Inclusive sorts before non-inclusive. + * + * @param o range to compare + * @return comparison result */ + @Override public int compareTo(Range o) { int comp; http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/test/src/main/java/org/apache/accumulo/test/randomwalk/Environment.java ---------------------------------------------------------------------- diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Environment.java index b99ee77,0000000..72792ef mode 100644,000000..100644 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Environment.java +++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Environment.java @@@ -1,215 -1,0 +1,213 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.randomwalk; + +import static com.google.common.base.Preconditions.checkNotNull; ++ +import java.lang.management.ManagementFactory; +import java.util.Properties; +import java.util.concurrent.TimeUnit; ++ +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.ClientConfiguration; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.MultiTableBatchWriter; +import org.apache.accumulo.core.client.ZooKeeperInstance; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.log4j.Logger; + +/** + * The test environment that is available for randomwalk tests. This includes configuration properties that are available to any randomwalk test and facilities + * for creating client-side objects. This class is not thread-safe. + */ +public class Environment { + /** + * The configuration property key for a username. + */ + public static final String KEY_USERNAME = "USERNAME"; + /** + * The configuration property key for a password. + */ + public static final String KEY_PASSWORD = "PASSWORD"; + /** + * The configuration property key for the instance name. + */ + public static final String KEY_INSTANCE = "INSTANCE"; + /** + * The configuration property key for the comma-separated list of ZooKeepers. + */ + public static final String KEY_ZOOKEEPERS = "ZOOKEEPERS"; + /** + * The configuration property key for the maximum memory for the multi-table batch writer. + */ + public static final String KEY_MAX_MEM = "MAX_MEM"; + /** + * The configuration property key for the maximum latency, in milliseconds, for the multi-table batch writer. + */ + public static final String KEY_MAX_LATENCY = "MAX_LATENCY"; + /** + * The configuration property key for the number of write threads for the multi-table batch writer. + */ + public static final String KEY_NUM_THREADS = "NUM_THREADS"; + + private static final Logger log = Logger.getLogger(Environment.class); + + private final Properties p; + private Instance instance = null; + private Connector connector = null; + private MultiTableBatchWriter mtbw = null; + + /** + * Creates a new test environment. + * + * @param p + * configuration properties + * @throws NullPointerException + * if p is null + */ + Environment(Properties p) { + checkNotNull(p); + this.p = p; + } + + /** + * Gets a copy of the configuration properties. + * + * @return a copy of the configuration properties + */ + Properties copyConfigProperties() { + return new Properties(p); + } + + /** + * Gets a configuration property. + * + * @param key + * key + * @return property value + */ + public String getConfigProperty(String key) { + return p.getProperty(key); + } + + /** + * Gets the configured username. + * + * @return username + */ + public String getUserName() { + return p.getProperty(KEY_USERNAME); + } + + /** + * Gets the configured password. + * + * @return password + */ + public String getPassword() { + return p.getProperty(KEY_PASSWORD); + } + + /** + * Gets this process's ID. + * + * @return pid + */ + public String getPid() { + return ManagementFactory.getRuntimeMXBean().getName().split("@")[0]; + } + + /** + * Gets an authentication token based on the configured password. + * + * @return authentication token + */ + public AuthenticationToken getToken() { + return new PasswordToken(getPassword()); + } + + /** + * Gets an Accumulo instance object. The same instance is reused after the first call. + * + * @return instance + */ + public Instance getInstance() { + if (instance == null) { + String instance = p.getProperty(KEY_INSTANCE); + String zookeepers = p.getProperty(KEY_ZOOKEEPERS); + this.instance = new ZooKeeperInstance(new ClientConfiguration().withInstance(instance).withZkHosts(zookeepers)); + } + return instance; + } + + /** + * Gets an Accumulo connector. The same connector is reused after the first call. + * + * @return connector - * @throws AccumuloException - * @throws AccumuloSecurityException + */ + public Connector getConnector() throws AccumuloException, AccumuloSecurityException { + if (connector == null) { + connector = getInstance().getConnector(getUserName(), getToken()); + } + return connector; + } + + /** + * Gets a multitable batch writer. The same object is reused after the first call unless it is reset. + * + * @return multitable batch writer + * @throws NumberFormatException + * if any of the numeric batch writer configuration properties cannot be parsed - * @throws AccumuloException - * @throws AccumuloSecurityException + * @throws NumberFormatException + * if any configuration property cannot be parsed + */ + public MultiTableBatchWriter getMultiTableBatchWriter() throws AccumuloException, AccumuloSecurityException { + if (mtbw == null) { + long maxMem = Long.parseLong(p.getProperty(KEY_MAX_MEM)); + long maxLatency = Long.parseLong(p.getProperty(KEY_MAX_LATENCY)); + int numThreads = Integer.parseInt(p.getProperty(KEY_NUM_THREADS)); + mtbw = getConnector().createMultiTableBatchWriter( + new BatchWriterConfig().setMaxMemory(maxMem).setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(numThreads)); + } + return mtbw; + } + + /** + * Checks if a multitable batch writer has been created by this wrapper. + * + * @return true if multitable batch writer is already created + */ + public boolean isMultiTableBatchWriterInitialized() { + return mtbw != null; + } + + /** + * Clears the multitable batch writer previously created and remembered by this wrapper. + */ + public void resetMultiTableBatchWriter() { + if (mtbw == null) + return; + if (!mtbw.isClosed()) { + log.warn("Setting non-closed MultiTableBatchWriter to null (leaking resources)"); + } + mtbw = null; + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java ---------------------------------------------------------------------- diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java index 56fb366,7cb58c9..0e54d90 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java +++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java @@@ -49,10 -53,8 +49,8 @@@ public class Framework * * @param startName * Full name of starting graph or test - * @param state - * @param confDir */ - public int run(String startName, State state, String confDir) { + public int run(String startName, State state, Environment env, String confDir) { try { System.out.println("confDir " + confDir); http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java ---------------------------------------------------------------------- diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java index 980c608,b74b6cd..1588d5a --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java +++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java @@@ -34,11 -33,8 +34,10 @@@ public abstract class Node * * @param state * Random walk state passed between nodes + * @param env + * test environment - * @throws Exception */ - public abstract void visit(State state, Properties props) throws Exception; + public abstract void visit(State state, Environment env, Properties props) throws Exception; @Override public boolean equals(Object o) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java ---------------------------------------------------------------------- diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java index 0335b49,8dc2b6e..df246d4 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java +++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java @@@ -40,14 -39,11 +40,11 @@@ public class CheckBalance extends Test static final String LAST_UNBALANCED_TIME = "lastUnbalancedTime"; static final String UNBALANCED_COUNT = "unbalancedCount"; - /* (non-Javadoc) - * @see org.apache.accumulo.test.randomwalk.Node#visit(org.apache.accumulo.test.randomwalk.State, java.util.Properties) - */ @Override - public void visit(State state, Properties props) throws Exception { + public void visit(State state, Environment env, Properties props) throws Exception { log.debug("checking balance"); Map<String,Long> counts = new HashMap<String,Long>(); - Scanner scanner = state.getConnector().createScanner(MetadataTable.NAME, Authorizations.EMPTY); + Scanner scanner = env.getConnector().createScanner(MetadataTable.NAME, Authorizations.EMPTY); scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME); for (Entry<Key,Value> entry : scanner) { String location = entry.getKey().getColumnQualifier().toString();