Repository: gora Updated Branches: refs/heads/master a40f72f98 -> 356c1f8a8
GORA-414 Switch to Accumulo 1.6.4 and switched a test to MiniAccumulo Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/356c1f8a Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/356c1f8a Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/356c1f8a Branch: refs/heads/master Commit: 356c1f8a825f883c8de43a3e25019ff230a26457 Parents: a40f72f Author: Keith Turner <[email protected]> Authored: Mon Nov 16 18:43:46 2015 -0500 Committer: Keith Turner <[email protected]> Committed: Mon Nov 16 21:14:08 2015 -0500 ---------------------------------------------------------------------- gora-accumulo/pom.xml | 10 ++++- .../gora/accumulo/store/AccumuloStore.java | 39 ++++++++++++-------- .../gora/accumulo/GoraAccumuloTestDriver.java | 8 +++- .../src/test/resources/gora.properties | 8 ++-- 4 files changed, 43 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/356c1f8a/gora-accumulo/pom.xml ---------------------------------------------------------------------- diff --git a/gora-accumulo/pom.xml b/gora-accumulo/pom.xml index 8ab5143..bc131fa 100644 --- a/gora-accumulo/pom.xml +++ b/gora-accumulo/pom.xml @@ -50,7 +50,7 @@ </ciManagement> <properties> - <accumulo.version>1.5.1</accumulo.version> + <accumulo.version>1.6.4</accumulo.version> <osgi.import>*</osgi.import> <osgi.export>org.apache.gora.accumulo*;version="${project.version}";-noimport:=true</osgi.export> </properties> @@ -127,6 +127,14 @@ </dependency> <dependency> + <!-- Accumulo mini was trying to load the Shell, which failed with the version of jline in the parent pom --> + <groupId>jline</groupId> + <artifactId>jline</artifactId> + <version>2.11</version> + <scope>test</scope> + </dependency> + + <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/gora/blob/356c1f8a/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java ---------------------------------------------------------------------- diff --git a/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java b/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java index 31141c3..e6b6e65 100644 --- a/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java +++ b/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java @@ -35,7 +35,6 @@ import java.util.concurrent.TimeUnit; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; -import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; @@ -67,11 +66,10 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.SortedKeyIterator; import org.apache.accumulo.core.iterators.user.TimestampFilter; import org.apache.accumulo.core.master.state.tables.TableState; +import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.accumulo.core.security.CredentialHelper; -import org.apache.accumulo.core.security.thrift.TCredentials; +import org.apache.accumulo.core.security.Credentials; import org.apache.accumulo.core.util.Pair; -import org.apache.accumulo.core.util.TextUtil; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.avro.Schema; import org.apache.avro.Schema.Field; @@ -121,7 +119,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T private Connector conn; private BatchWriter batchWriter; private AccumuloMapping mapping; - private TCredentials credentials; + private Credentials credentials; private Encoder encoder; public static final Logger LOG = LoggerFactory.getLogger(AccumuloStore.class); @@ -188,6 +186,15 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T } + private static byte[] getBytes(Text text) { + byte[] bytes = text.getBytes(); + if (bytes.length != text.getLength()) { + bytes = new byte[text.getLength()]; + System.arraycopy(text.getBytes(), 0, bytes, 0, bytes.length); + } + return bytes; + } + public K fromBytes(Class<K> clazz, byte[] val) { return fromBytes(encoder, clazz, val); } @@ -363,7 +370,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T } else { conn = new MockInstance().getConnector(user, token); } - credentials = CredentialHelper.create(user, token, conn.getInstance().getInstanceID()); + credentials = new Credentials(user, token); if (autoCreateSchema && !schemaExists()) createSchema(); @@ -452,8 +459,10 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T conn.tableOperations().setProperty(mapping.tableName, entry.getKey(), entry.getValue()); } - } catch (AccumuloException | AccumuloSecurityException | TableExistsException e) { + } catch (AccumuloException | AccumuloSecurityException e) { LOG.error(e.getMessage(), e); + } catch (TableExistsException e) { + LOG.debug(e.getMessage(), e); } } @@ -614,7 +623,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T public T get(K key, String[] fields) { try { // TODO make isolated scanner optional? - Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Constants.NO_AUTHS)); + Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Authorizations.EMPTY)); Range rowRange = new Range(new Text(toBytes(key))); scanner.setRange(rowRange); @@ -818,7 +827,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T private Scanner createScanner(Query<K,T> query) throws TableNotFoundException { // TODO make isolated scanner optional? - Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Constants.NO_AUTHS)); + Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Authorizations.EMPTY)); setFetchColumns(scanner, query.getFields()); scanner.setRange(createRange(query)); @@ -871,12 +880,12 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T if (conn instanceof MockConnector) tl = new MockTabletLocator(); else - tl = TabletLocator.getInstance(conn.getInstance(), new Text(Tables.getTableId(conn.getInstance(), mapping.tableName))); + tl = TabletLocator.getLocator(conn.getInstance(), new Text(Tables.getTableId(conn.getInstance(), mapping.tableName))); Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<>(); tl.invalidateCache(); - while (tl.binRanges(Collections.singletonList(createRange(query)), binnedRanges, credentials).size() > 0) { + while (tl.binRanges(credentials, Collections.singletonList(createRange(query)), binnedRanges).size() > 0) { // TODO log? if (!Tables.exists(conn.getInstance(), Tables.getTableId(conn.getInstance(), mapping.tableName))) throw new TableDeletedException(Tables.getTableId(conn.getInstance(), mapping.tableName)); @@ -913,18 +922,18 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T K startKey = null; if (startRow == null || !ke.contains(startRow)) { if (ke.getPrevEndRow() != null) { - startKey = followingKey(encoder, getKeyClass(), TextUtil.getBytes(ke.getPrevEndRow())); + startKey = followingKey(encoder, getKeyClass(), getBytes(ke.getPrevEndRow())); } } else { - startKey = fromBytes(getKeyClass(), TextUtil.getBytes(startRow)); + startKey = fromBytes(getKeyClass(), getBytes(startRow)); } K endKey = null; if (endRow == null || !ke.contains(endRow)) { if (ke.getEndRow() != null) - endKey = lastPossibleKey(encoder, getKeyClass(), TextUtil.getBytes(ke.getEndRow())); + endKey = lastPossibleKey(encoder, getKeyClass(), getBytes(ke.getEndRow())); } else { - endKey = fromBytes(getKeyClass(), TextUtil.getBytes(endRow)); + endKey = fromBytes(getKeyClass(), getBytes(endRow)); } PartitionQueryImpl<K, T> pqi = new PartitionQueryImpl<>(query, startKey, endKey, location); http://git-wip-us.apache.org/repos/asf/gora/blob/356c1f8a/gora-accumulo/src/test/java/org/apache/gora/accumulo/GoraAccumuloTestDriver.java ---------------------------------------------------------------------- diff --git a/gora-accumulo/src/test/java/org/apache/gora/accumulo/GoraAccumuloTestDriver.java b/gora-accumulo/src/test/java/org/apache/gora/accumulo/GoraAccumuloTestDriver.java index 0d33444..bd3b502 100644 --- a/gora-accumulo/src/test/java/org/apache/gora/accumulo/GoraAccumuloTestDriver.java +++ b/gora-accumulo/src/test/java/org/apache/gora/accumulo/GoraAccumuloTestDriver.java @@ -18,6 +18,7 @@ package org.apache.gora.accumulo; import org.apache.accumulo.minicluster.MiniAccumuloCluster; +import org.apache.accumulo.minicluster.MiniAccumuloConfig; import org.apache.gora.GoraTestDriver; import org.apache.gora.accumulo.store.AccumuloStore; import org.junit.Rule; @@ -33,7 +34,7 @@ public class GoraAccumuloTestDriver extends GoraTestDriver { private static final Logger LOG = LoggerFactory.getLogger(GoraAccumuloTestDriver.class); private static MiniAccumuloCluster cluster = null; - private static final String PASSWORD = "password"; + private static final String PASSWORD = "drowssap"; @Rule public TemporaryFolder tmpDir = new TemporaryFolder(); @@ -48,7 +49,10 @@ public class GoraAccumuloTestDriver extends GoraTestDriver { log.info("Starting Accumulo MiniAccumuloCluster..."); try { tmpDir.create(); - cluster = new MiniAccumuloCluster(tmpDir.getRoot(), PASSWORD); + MiniAccumuloConfig miniCfg = new MiniAccumuloConfig(tmpDir.getRoot(), PASSWORD); + miniCfg.setInstanceName("goraTest"); + miniCfg.setZooKeeperPort(56321); + cluster = new MiniAccumuloCluster(miniCfg); cluster.start(); } catch (Exception e) { LOG.error("Error starting Accumulo MiniAccumuloCluster: {}", e.getMessage()); http://git-wip-us.apache.org/repos/asf/gora/blob/356c1f8a/gora-accumulo/src/test/resources/gora.properties ---------------------------------------------------------------------- diff --git a/gora-accumulo/src/test/resources/gora.properties b/gora-accumulo/src/test/resources/gora.properties index f89a360..8d1c4f5 100644 --- a/gora-accumulo/src/test/resources/gora.properties +++ b/gora-accumulo/src/test/resources/gora.properties @@ -14,8 +14,8 @@ # limitations under the License. gora.datastore.default=org.apache.gora.accumulo.store.AccumuloStore -gora.datastore.accumulo.mock=true -gora.datastore.accumulo.instance=a14 -gora.datastore.accumulo.zookeepers=localhost +gora.datastore.accumulo.mock=false +gora.datastore.accumulo.instance=goraTest +gora.datastore.accumulo.zookeepers=localhost:56321 gora.datastore.accumulo.user=root -gora.datastore.accumulo.password= \ No newline at end of file +gora.datastore.accumulo.password=drowssap \ No newline at end of file
