Updated Branches: refs/heads/ACCUMULO-1833-caching 0be6f0a79 -> 29efac97c
ACCUMULO-1833 Remove Connector client methods, but leave constructor on MTBW in place for testing purposes. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c8e607fe Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c8e607fe Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c8e607fe Branch: refs/heads/ACCUMULO-1833-caching Commit: c8e607fe381242492fc7e8c84b4fb0f1e000d3aa Parents: 0be6f0a Author: Josh Elser <josh.el...@gmail.com> Authored: Fri Nov 8 10:55:01 2013 -0500 Committer: Josh Elser <josh.el...@gmail.com> Committed: Fri Nov 8 10:56:24 2013 -0500 ---------------------------------------------------------------------- .../apache/accumulo/core/client/Connector.java | 44 +------------------ .../core/client/impl/ConnectorImpl.java | 12 ----- .../client/impl/MultiTableBatchWriterImpl.java | 4 +- .../core/client/mock/MockConnector.java | 11 ----- .../test/MultiTableBatchWriterTest.java | 46 ++++++++++++-------- 5 files changed, 31 insertions(+), 86 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/c8e607fe/core/src/main/java/org/apache/accumulo/core/client/Connector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/Connector.java b/core/src/main/java/org/apache/accumulo/core/client/Connector.java index 68dc881..d2e7321 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/Connector.java +++ b/core/src/main/java/org/apache/accumulo/core/client/Connector.java @@ -16,8 +16,6 @@ */ package org.apache.accumulo.core.client; -import java.util.concurrent.TimeUnit; - import org.apache.accumulo.core.client.admin.InstanceOperations; import org.apache.accumulo.core.client.admin.SecurityOperations; import org.apache.accumulo.core.client.admin.TableOperations; @@ -148,32 +146,8 @@ public abstract class Connector { public abstract MultiTableBatchWriter createMultiTableBatchWriter(long maxMemory, long maxLatency, int maxWriteThreads); /** - * Factory method to create a Multi-Table BatchWriter connected to Accumulo. Multi-table batch writers can queue data for multiple tables, which is good for - * ingesting data into multiple tables from the same source. Caching of ZooKeeper table information defaults to {@link MultiTableBatchWriterImpl#DEFAULT_CACHE_TIME} - * and {@link MultiTableBatchWriterImpl#DEFAULT_CACHE_TIME_UNIT} - * - * @param maxMemory - * size in bytes of the maximum memory to batch before writing - * @param maxLatency - * size in milliseconds; set to 0 or Long.MAX_VALUE to allow the maximum time to hold a batch before writing - * @param maxWriteThreads - * the maximum number of threads to use for writing data to the tablet servers - * @param cacheTime - * Duration of time to cache ZooKeeper table information - * @param cacheTimeUnit - * Unit of time to apply to {@link cacheTime} - * - * @return MultiTableBatchWriter object for configuring and writing data to - * @deprecated since 1.5.0; Use {@link #createMultiTableBatchWriter(BatchWriterConfig)} instead. - * @since 1.5.1 - */ - @Deprecated - public abstract MultiTableBatchWriter createMultiTableBatchWriter(long maxMemory, long maxLatency, int maxWriteThreads, long cacheTime, TimeUnit cacheTimeUnit); - - /** * Factory method to create a Multi-Table BatchWriter connected to Accumulo. Multi-table batch writers can queue data for multiple tables. Also data for - * multiple tables can be sent to a server in a single batch. Its an efficient way to ingest data into multiple tables from a single process. Caching - * of ZooKeeper table information defaults to {@link MultiTableBatchWriterImpl#DEFAULT_CACHE_TIME} and {@link MultiTableBatchWriterImpl#DEFAULT_CACHE_TIME_UNIT} + * multiple tables can be sent to a server in a single batch. Its an efficient way to ingest data into multiple tables from a single process. * * @param config * configuration used to create multi-table batch writer @@ -184,22 +158,6 @@ public abstract class Connector { public abstract MultiTableBatchWriter createMultiTableBatchWriter(BatchWriterConfig config); /** - * Factory method to create a Multi-Table BatchWriter connected to Accumulo. Multi-table batch writers can queue data for multiple tables. Also data for - * multiple tables can be sent to a server in a single batch. Its an efficient way to ingest data into multiple tables from a single process. This method - * also allows the user to provide parameters as to how long table information from ZooKeeper is cached. - * @param config - * configuration used to create the multi-table batch writer - * @param cacheTime - * Duration of time to cache ZooKeeper table information - * @param cacheTimeUnit - * Unit of time to apply to {@link cacheTime} - * @return - * @since 1.5.1 - */ - public abstract MultiTableBatchWriter createMultiTableBatchWriter(BatchWriterConfig config, long cacheTime, TimeUnit cacheTimeUnit); - - - /** * Factory method to create a Scanner connected to Accumulo. * * @param tableName http://git-wip-us.apache.org/repos/asf/accumulo/blob/c8e607fe/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java b/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java index 89d2813..1702082 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java @@ -126,24 +126,12 @@ public class ConnectorImpl extends Connector { .setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(maxWriteThreads)); } - @Deprecated - @Override - public MultiTableBatchWriter createMultiTableBatchWriter(long maxMemory, long maxLatency, int maxWriteThreads, long cacheTime, TimeUnit cacheTimeUnit) { - return new MultiTableBatchWriterImpl(instance, credentials, new BatchWriterConfig().setMaxMemory(maxMemory) - .setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(maxWriteThreads), cacheTime, cacheTimeUnit); - } - @Override public MultiTableBatchWriter createMultiTableBatchWriter(BatchWriterConfig config) { return new MultiTableBatchWriterImpl(instance, credentials, config); } @Override - public MultiTableBatchWriter createMultiTableBatchWriter(BatchWriterConfig config, long timeToCache, TimeUnit timeUnit) { - return new MultiTableBatchWriterImpl(instance, credentials, config, timeToCache, timeUnit); - } - - @Override public Scanner createScanner(String tableName, Authorizations authorizations) throws TableNotFoundException { ArgumentChecker.notNull(tableName, authorizations); return new ScannerImpl(instance, credentials, getTableId(tableName), authorizations); http://git-wip-us.apache.org/repos/asf/accumulo/blob/c8e607fe/core/src/main/java/org/apache/accumulo/core/client/impl/MultiTableBatchWriterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/MultiTableBatchWriterImpl.java b/core/src/main/java/org/apache/accumulo/core/client/impl/MultiTableBatchWriterImpl.java index d292379..34318d1 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/impl/MultiTableBatchWriterImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/client/impl/MultiTableBatchWriterImpl.java @@ -41,8 +41,8 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; public class MultiTableBatchWriterImpl implements MultiTableBatchWriter { - public static final long DEFAULT_CACHE_TIME = 60; - public static final TimeUnit DEFAULT_CACHE_TIME_UNIT = TimeUnit.SECONDS; + public static final long DEFAULT_CACHE_TIME = 200; + public static final TimeUnit DEFAULT_CACHE_TIME_UNIT = TimeUnit.MILLISECONDS; static final Logger log = Logger.getLogger(MultiTableBatchWriterImpl.class); private AtomicBoolean closed; http://git-wip-us.apache.org/repos/asf/accumulo/blob/c8e607fe/core/src/main/java/org/apache/accumulo/core/client/mock/MockConnector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockConnector.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockConnector.java index 2aa6291..1179559 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockConnector.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockConnector.java @@ -90,23 +90,12 @@ public class MockConnector extends Connector { return new MockMultiTableBatchWriter(acu); } - @Deprecated - @Override - public MultiTableBatchWriter createMultiTableBatchWriter(long maxMemory, long maxLatency, int maxWriteThreads, long cacheTime, TimeUnit cacheTimeUnit) { - return new MockMultiTableBatchWriter(acu); - } - @Override public MultiTableBatchWriter createMultiTableBatchWriter(BatchWriterConfig config) { return createMultiTableBatchWriter(config.getMaxMemory(), config.getMaxLatency(TimeUnit.MILLISECONDS), config.getMaxWriteThreads()); } @Override - public MultiTableBatchWriter createMultiTableBatchWriter(BatchWriterConfig config, long cacheTime, TimeUnit cacheTimeUnit) { - return createMultiTableBatchWriter(config.getMaxMemory(), config.getMaxLatency(TimeUnit.MILLISECONDS), config.getMaxWriteThreads(), cacheTime, cacheTimeUnit); - } - - @Override public Scanner createScanner(String tableName, Authorizations authorizations) throws TableNotFoundException { MockTable table = acu.tables.get(tableName); if (table == null) http://git-wip-us.apache.org/repos/asf/accumulo/blob/c8e607fe/test/src/test/java/org/apache/accumulo/test/MultiTableBatchWriterTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/MultiTableBatchWriterTest.java b/test/src/test/java/org/apache/accumulo/test/MultiTableBatchWriterTest.java index 2cb78ed..8ed6931 100644 --- a/test/src/test/java/org/apache/accumulo/test/MultiTableBatchWriterTest.java +++ b/test/src/test/java/org/apache/accumulo/test/MultiTableBatchWriterTest.java @@ -32,12 +32,15 @@ import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.TableOfflineException; import org.apache.accumulo.core.client.ZooKeeperInstance; import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.core.client.impl.MultiTableBatchWriterImpl; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.security.CredentialHelper; +import org.apache.accumulo.core.security.thrift.TCredentials; import org.apache.accumulo.minicluster.MiniAccumuloCluster; import org.apache.accumulo.minicluster.MiniAccumuloConfig; import org.junit.AfterClass; @@ -52,12 +55,12 @@ import com.google.common.util.concurrent.UncheckedExecutionException; public class MultiTableBatchWriterTest { public static TemporaryFolder folder = new TemporaryFolder(); public static MiniAccumuloCluster cluster; - private static final String password = "secret"; + private static final PasswordToken password = new PasswordToken("secret"); @BeforeClass public static void setUpBeforeClass() throws Exception { folder.create(); - MiniAccumuloConfig cfg = new MiniAccumuloConfig(folder.newFolder("miniAccumulo"), password); + MiniAccumuloConfig cfg = new MiniAccumuloConfig(folder.newFolder("miniAccumulo"), new String(password.getPassword())); cluster = new MiniAccumuloCluster(cfg); cluster.start(); } @@ -71,11 +74,12 @@ public class MultiTableBatchWriterTest { @Test public void testTableRenameSameWriters() throws Exception { ZooKeeperInstance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()); - Connector connector = instance.getConnector("root", new PasswordToken(password)); + Connector connector = instance.getConnector("root", password); BatchWriterConfig config = new BatchWriterConfig(); - MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(config, 60, TimeUnit.SECONDS); + TCredentials creds = CredentialHelper.create("root", password, instance.getInstanceID()); + MultiTableBatchWriter mtbw = new MultiTableBatchWriterImpl(instance, creds, config, 60, TimeUnit.SECONDS); try { final String table1 = "testTableRenameSameWriters_table1", table2 = "testTableRenameSameWriters_table2"; @@ -132,11 +136,12 @@ public class MultiTableBatchWriterTest { @Test public void testTableRenameNewWriters() throws Exception { ZooKeeperInstance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()); - Connector connector = instance.getConnector("root", new PasswordToken(password)); + Connector connector = instance.getConnector("root", password); BatchWriterConfig config = new BatchWriterConfig(); - MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(config, 60, TimeUnit.MINUTES); + TCredentials creds = CredentialHelper.create("root", password, instance.getInstanceID()); + MultiTableBatchWriter mtbw = new MultiTableBatchWriterImpl(instance, creds, config, 60, TimeUnit.SECONDS); try { final String table1 = "testTableRenameNewWriters_table1", table2 = "testTableRenameNewWriters_table2"; @@ -197,11 +202,12 @@ public class MultiTableBatchWriterTest { @Test public void testTableRenameNewWritersNoCaching() throws Exception { ZooKeeperInstance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()); - Connector connector = instance.getConnector("root", new PasswordToken(password)); + Connector connector = instance.getConnector("root", password); BatchWriterConfig config = new BatchWriterConfig(); - - MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(config, 0, TimeUnit.SECONDS); + + TCredentials creds = CredentialHelper.create("root", password, instance.getInstanceID()); + MultiTableBatchWriter mtbw = new MultiTableBatchWriterImpl(instance, creds, config, 0, TimeUnit.SECONDS); try { final String table1 = "testTableRenameNewWritersNoCaching_table1", table2 = "testTableRenameNewWritersNoCaching_table2"; @@ -245,11 +251,12 @@ public class MultiTableBatchWriterTest { @Test public void testTableDelete() throws Exception { ZooKeeperInstance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()); - Connector connector = instance.getConnector("root", new PasswordToken(password)); + Connector connector = instance.getConnector("root", password); BatchWriterConfig config = new BatchWriterConfig(); - MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(config, 60, TimeUnit.MINUTES); + TCredentials creds = CredentialHelper.create("root", password, instance.getInstanceID()); + MultiTableBatchWriter mtbw = new MultiTableBatchWriterImpl(instance, creds, config, 60, TimeUnit.SECONDS); try { final String table1 = "testTableDelete_table1", table2 = "testTableDelete_table2"; @@ -294,11 +301,12 @@ public class MultiTableBatchWriterTest { public void testOfflineTable() throws Exception { ZooKeeperInstance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()); - Connector connector = instance.getConnector("root", new PasswordToken(password)); + Connector connector = instance.getConnector("root", password); BatchWriterConfig config = new BatchWriterConfig(); - MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(config, 60, TimeUnit.MINUTES); + TCredentials creds = CredentialHelper.create("root", password, instance.getInstanceID()); + MultiTableBatchWriter mtbw = new MultiTableBatchWriterImpl(instance, creds, config, 60, TimeUnit.SECONDS); try { final String table1 = "testOfflineTable_table1", table2 = "testOfflineTable_table2"; @@ -344,11 +352,12 @@ public class MultiTableBatchWriterTest { public void testOfflineTableWithCache() throws Exception { ZooKeeperInstance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()); - Connector connector = instance.getConnector("root", new PasswordToken(password)); + Connector connector = instance.getConnector("root", password); BatchWriterConfig config = new BatchWriterConfig(); - - MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(config, 60, TimeUnit.MINUTES); + + TCredentials creds = CredentialHelper.create("root", password, instance.getInstanceID()); + MultiTableBatchWriter mtbw = new MultiTableBatchWriterImpl(instance, creds, config, 60, TimeUnit.SECONDS); try { final String table1 = "testOfflineTableWithCache_table1", table2 = "testOfflineTableWithCache_table2"; @@ -396,11 +405,12 @@ public class MultiTableBatchWriterTest { public void testOfflineTableWithoutCache() throws Exception { ZooKeeperInstance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()); - Connector connector = instance.getConnector("root", new PasswordToken(password)); + Connector connector = instance.getConnector("root", password); BatchWriterConfig config = new BatchWriterConfig(); - MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(config, 0, TimeUnit.MINUTES); + TCredentials creds = CredentialHelper.create("root", password, instance.getInstanceID()); + MultiTableBatchWriter mtbw = new MultiTableBatchWriterImpl(instance, creds, config, 0, TimeUnit.SECONDS); try { final String table1 = "testOfflineTableWithoutCache_table1", table2 = "testOfflineTableWithoutCache_table2";
