Repository: gora Updated Branches: refs/heads/master 3cacf2a25 -> 92b884158
GORA-353 Accumulo authentication token serialized incorrectly Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/92b88415 Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/92b88415 Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/92b88415 Branch: refs/heads/master Commit: 92b88415820001da61e0a94aa526db3622b576dc Parents: 3cacf2a Author: Lewis John McGibbney <[email protected]> Authored: Fri Aug 15 11:46:43 2014 -0700 Committer: Lewis John McGibbney <[email protected]> Committed: Fri Aug 15 11:46:43 2014 -0700 ---------------------------------------------------------------------- CHANGES.txt | 2 + gora-accumulo/pom.xml | 304 ++++++++++--------- .../gora/accumulo/store/AccumuloStore.java | 42 ++- .../accumulo/store/AuthenticationTokenTest.java | 90 ++++++ 4 files changed, 275 insertions(+), 163 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/92b88415/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index c1fca0c..808b674 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,8 @@ Current Development 0.5-SNAPSHOT +* GORA-353 Accumulo authentication token serialized incorrectly (Chin Huang via lewismc) + * GORA-167 forward port of Make Cassandra keyspace consistency configurable within gora.properties (rmarroquin via lewismc) * GORA-364 MemStore.get fails with NPE when key is not set (Gerhard Gossen via lewismc) http://git-wip-us.apache.org/repos/asf/gora/blob/92b88415/gora-accumulo/pom.xml ---------------------------------------------------------------------- diff --git a/gora-accumulo/pom.xml b/gora-accumulo/pom.xml index ff0956b..b588a7d 100644 --- a/gora-accumulo/pom.xml +++ b/gora-accumulo/pom.xml @@ -1,145 +1,169 @@ <?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <!-- Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with this - work for additional information regarding copyright ownership. The ASF licenses - this file to You under the Apache License, Version 2.0 (the "License"); you - may not use this file except in compliance with the License. You may obtain - a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless - required by applicable law or agreed to in writing, software distributed - under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES - OR CONDITIONS OF ANY KIND, either express or implied. See the License for - the specific language governing permissions and limitations under the License. --> - - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>org.apache.gora</groupId> - <artifactId>gora</artifactId> - <version>0.5-SNAPSHOT</version> - <relativePath>../</relativePath> - </parent> - <artifactId>gora-accumulo</artifactId> - <packaging>bundle</packaging> - - <name>Apache Gora :: Accumulo</name> - <url>http://gora.apache.org</url> - <description>The Apache Gora open source framework provides an in-memory data model and - persistence for big data. Gora supports persisting to column stores, key value stores, - document stores and RDBMSs, and analyzing the data with extensive Apache Hadoop MapReduce + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.gora</groupId> + <artifactId>gora</artifactId> + <version>0.5-SNAPSHOT</version> + <relativePath>../</relativePath> + </parent> + <artifactId>gora-accumulo</artifactId> + <packaging>bundle</packaging> + + <name>Apache Gora :: Accumulo</name> + <url>http://gora.apache.org</url> + <description>The Apache Gora open source framework provides an in-memory data model and + persistence for big data. Gora supports persisting to column stores, key value stores, + document stores and RDBMSs, and analyzing the data with extensive Apache Hadoop MapReduce support.</description> - <inceptionYear>2010</inceptionYear> - <organization> - <name>The Apache Software Foundation</name> - <url>http://www.apache.org/</url> - </organization> - <issueManagement> - <system>JIRA</system> - <url>https://issues.apache.org/jira/browse/GORA</url> - </issueManagement> - <ciManagement> - <system>Jenkins</system> - <url>https://builds.apache.org/job/Gora-trunk/</url> - </ciManagement> - - <properties> - <osgi.import>*</osgi.import> - <osgi.export>org.apache.gora.accumulo*;version="${project.version}";-noimport:=true</osgi.export> - </properties> - - <build> - <directory>target</directory> - <outputDirectory>target/classes</outputDirectory> - <finalName>${project.artifactId}-${project.version}</finalName> - <testOutputDirectory>target/test-classes</testOutputDirectory> - <testSourceDirectory>src/test/java</testSourceDirectory> - <sourceDirectory>src/main/java</sourceDirectory> - <testResources> - <testResource> - <directory>${project.basedir}/src/test/resources</directory> - <includes> - <include>**/*</include> - </includes> - <!--targetPath>${project.basedir}/target/classes/</targetPath--> - </testResource> - </testResources> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <version>${build-helper-maven-plugin.version}</version> - <executions> - <execution> - <phase>generate-sources</phase> - <goals> - <goal>add-source</goal> - </goals> - <configuration> - <sources> - <source>src/examples/java</source> - </sources> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - - <dependencies> - <!-- Gora Internal Dependencies --> - <dependency> - <groupId>org.apache.gora</groupId> - <artifactId>gora-core</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.gora</groupId> - <artifactId>gora-core</artifactId> - <type>test-jar</type> - <scope>test</scope> - </dependency> - - <!--Accumulo Dependency --> - <dependency> - <groupId>org.apache.accumulo</groupId> - <artifactId>accumulo-core</artifactId> - <version>1.5.1</version> - </dependency> - - <!-- Hadoop Dependencies --> - <dependency> - <groupId>org.apache.avro</groupId> - <artifactId>avro</artifactId> - </dependency> - - <dependency> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - <exclusions> - <exclusion> - <groupId>javax.jms</groupId> - <artifactId>jms</artifactId> - </exclusion> - </exclusions> - </dependency> - - <!-- Testing Dependencies --> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-test</artifactId> - </dependency> - - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-simple</artifactId> - <scope>test</scope> - </dependency> - - </dependencies> + <inceptionYear>2010</inceptionYear> + <organization> + <name>The Apache Software Foundation</name> + <url>http://www.apache.org/</url> + </organization> + <issueManagement> + <system>JIRA</system> + <url>https://issues.apache.org/jira/browse/GORA</url> + </issueManagement> + <ciManagement> + <system>Jenkins</system> + <url>https://builds.apache.org/job/Gora-trunk/</url> + </ciManagement> + + <properties> + <accumulo.version>1.5.1</accumulo.version> + <osgi.import>*</osgi.import> + <osgi.export>org.apache.gora.accumulo*;version="${project.version}";-noimport:=true</osgi.export> + </properties> + + <build> + <directory>target</directory> + <outputDirectory>target/classes</outputDirectory> + <finalName>${project.artifactId}-${project.version}</finalName> + <testOutputDirectory>target/test-classes</testOutputDirectory> + <testSourceDirectory>src/test/java</testSourceDirectory> + <sourceDirectory>src/main/java</sourceDirectory> + <testResources> + <testResource> + <directory>${project.basedir}/src/test/resources</directory> + <includes> + <include>**/*</include> + </includes> + <!--targetPath>${project.basedir}/target/classes/</targetPath--> + </testResource> + </testResources> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>${build-helper-maven-plugin.version}</version> + <executions> + <execution> + <phase>generate-sources</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>src/examples/java</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + + <dependencies> + <!-- Gora Internal Dependencies --> + <dependency> + <groupId>org.apache.gora</groupId> + <artifactId>gora-core</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.gora</groupId> + <artifactId>gora-core</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + + <!--Accumulo Dependency --> + <dependency> + <groupId>org.apache.accumulo</groupId> + <artifactId>accumulo-core</artifactId> + <version>${accumulo.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.accumulo</groupId> + <artifactId>accumulo-minicluster</artifactId> + <version>${accumulo.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-io</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <scope>test</scope> + </dependency> + + <!-- Hadoop Dependencies --> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro</artifactId> + </dependency> + + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <exclusions> + <exclusion> + <groupId>javax.jms</groupId> + <artifactId>jms</artifactId> + </exclusion> + </exclusions> + </dependency> + + <!-- Testing Dependencies --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-test</artifactId> + </dependency> + + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-simple</artifactId> + <scope>test</scope> + </dependency> + + </dependencies> </project> http://git-wip-us.apache.org/repos/asf/gora/blob/92b88415/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java ---------------------------------------------------------------------- diff --git a/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java b/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java index 3c1911a..fff12be 100644 --- a/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java +++ b/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java @@ -68,6 +68,7 @@ import org.apache.accumulo.core.iterators.SortedKeyIterator; import org.apache.accumulo.core.iterators.user.TimestampFilter; import org.apache.accumulo.core.master.state.tables.TableState; import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.core.security.CredentialHelper; import org.apache.accumulo.core.security.thrift.TCredentials; import org.apache.accumulo.core.util.Pair; import org.apache.accumulo.core.util.TextUtil; @@ -106,7 +107,7 @@ import org.w3c.dom.Element; import org.w3c.dom.NodeList; /** - * + * Directs CRUD operations into Accumulo. */ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T> { @@ -250,7 +251,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T e.printStackTrace(); return toBytes(o); } - } else { + } else { return toBytes(o); } } @@ -357,20 +358,15 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T } try { - AuthenticationToken token = new PasswordToken(password); + AuthenticationToken token = new PasswordToken(password); if (mock == null || !mock.equals("true")) { String instance = DataStoreFactory.findProperty(properties, this, INSTANCE_NAME_PROPERTY, null); String zookeepers = DataStoreFactory.findProperty(properties, this, ZOOKEEPERS_NAME_PROPERTY, null); - credentials = new TCredentials(user, - "org.apache.accumulo.core.client.security.tokens.PasswordToken", - ByteBuffer.wrap(password.getBytes()), instance); conn = new ZooKeeperInstance(instance, zookeepers).getConnector(user, token); } else { - conn = new MockInstance().getConnector(user, new PasswordToken(password)); - credentials = new TCredentials(user, - "org.apache.accumulo.core.client.security.tokens.PasswordToken", - ByteBuffer.wrap(password.getBytes()), conn.getInstance().getInstanceID()); + conn = new MockInstance().getConnector(user, token); } + credentials = CredentialHelper.create(user, token, conn.getInstance().getInstanceID()); if (autoCreateSchema) createSchema(); @@ -379,7 +375,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T } catch (AccumuloSecurityException e) { throw new IOException(e); } - }catch(IOException e){ + } catch(IOException e){ LOG.error(e.getMessage(), e); } } @@ -483,7 +479,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T LOG.error(e.getMessage(), e); } catch (TableNotFoundException e) { LOG.error(e.getMessage(), e); - } + } } @Override @@ -515,7 +511,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T if (currentMap != null) { if (currentFam.equals(entry.getKey().getColumnFamily())) { - currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()), + currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()), fromBytes(currentSchema, entry.getValue().get())); continue; } else { @@ -539,7 +535,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T currentFam = entry.getKey().getColumnFamily(); currentSchema = field.schema().getValueType(); - currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()), + currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()), fromBytes(currentSchema, entry.getValue().get())); break; case ARRAY: @@ -572,7 +568,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T currentFam = entry.getKey().getColumnFamily(); currentSchema = effectiveSchema.getValueType(); - currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()), + currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()), fromBytes(currentSchema, entry.getValue().get())); break; } @@ -603,7 +599,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T * @return String The field name */ private String getFieldName(Entry<Key, Value> entry) { - String fieldName = mapping.columnMap.get(new Pair<Text,Text>(entry.getKey().getColumnFamily(), + String fieldName = mapping.columnMap.get(new Pair<Text,Text>(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier())); if (fieldName == null) { fieldName = mapping.columnMap.get(new Pair<Text,Text>(entry.getKey().getColumnFamily(), null)); @@ -667,7 +663,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T } Field field = fields.get(i); - Object o = val.get(field.pos()); + Object o = val.get(field.pos()); Pair<Text,Text> col = mapping.fieldMap.get(field.name()); @@ -735,11 +731,11 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T if (o == null){ return 0; } - + Set<?> es = ((Map<?, ?>)o).entrySet(); for (Object entry : es) { Object mapKey = ((Entry<?, ?>) entry).getKey(); - Object mapVal = ((Entry<?, ?>) entry).getValue(); + Object mapVal = ((Entry<?, ?>) entry).getValue(); if ((o instanceof DirtyMapWrapper && ((DirtyMapWrapper<?, ?>)o).isDirty()) || !(o instanceof DirtyMapWrapper)) { //mapVal instanceof Dirtyable && ((Dirtyable)mapVal).isDirty()) { m.put(col.getFirst(), new Text(toBytes(mapKey)), new Value(toBytes(valueType, mapVal))); @@ -763,7 +759,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T if (o == null){ return 0; } - + List<?> array = (List<?>) o; // both GenericArray and DirtyListWrapper int j = 0; for (Object item : array) { @@ -862,7 +858,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T // TODO return empty result? LOG.error(e.getMessage(), e); return null; - } + } } @Override @@ -1027,7 +1023,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T } } catch (MutationsRejectedException e) { LOG.error(e.getMessage(), e); - } + } } @Override @@ -1039,6 +1035,6 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T } } catch (MutationsRejectedException e) { LOG.error(e.getMessage(), e); - } + } } } http://git-wip-us.apache.org/repos/asf/gora/blob/92b88415/gora-accumulo/src/test/java/org/apache/gora/accumulo/store/AuthenticationTokenTest.java ---------------------------------------------------------------------- diff --git a/gora-accumulo/src/test/java/org/apache/gora/accumulo/store/AuthenticationTokenTest.java b/gora-accumulo/src/test/java/org/apache/gora/accumulo/store/AuthenticationTokenTest.java new file mode 100644 index 0000000..ecb8da9 --- /dev/null +++ b/gora-accumulo/src/test/java/org/apache/gora/accumulo/store/AuthenticationTokenTest.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gora.accumulo.store; + +import java.util.List; +import java.util.Properties; +import org.apache.accumulo.minicluster.MiniAccumuloCluster; +import org.apache.gora.examples.generated.Employee; +import org.apache.gora.query.PartitionQuery; +import org.apache.gora.store.DataStore; +import org.apache.gora.store.DataStoreFactory; +import org.apache.hadoop.conf.Configuration; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tests authentication token is serialized correctly. + */ +public class AuthenticationTokenTest { + private static final Logger LOG = LoggerFactory.getLogger(AuthenticationTokenTest.class); + + private static final String GORA_DATASTORE = + DataStoreFactory.GORA + "." + DataStoreFactory.DATASTORE + "."; + private static final String PASSWORD = "password"; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private MiniAccumuloCluster cluster; + private DataStore<String, Employee> employeeStore; + + @Before + @SuppressWarnings("unchecked") + public void setUp() throws Exception { + cluster = new MiniAccumuloCluster(temporaryFolder.getRoot(), PASSWORD); + cluster.start(); + + Properties properties = DataStoreFactory.createProps(); + properties.setProperty( + GORA_DATASTORE + AccumuloStore.MOCK_PROPERTY, + "false"); + properties.setProperty( + GORA_DATASTORE + AccumuloStore.INSTANCE_NAME_PROPERTY, + cluster.getInstanceName()); + properties.setProperty( + GORA_DATASTORE + AccumuloStore.ZOOKEEPERS_NAME_PROPERTY, + cluster.getZooKeepers()); + properties.setProperty( + GORA_DATASTORE + AccumuloStore.PASSWORD_PROPERTY, + PASSWORD); + + employeeStore = DataStoreFactory.createDataStore( + AccumuloStore.class, + String.class, + Employee.class, + new Configuration(), + properties); + } + + @After + public void tearDown() throws Exception { + cluster.stop(); + } + + @Test + public void testAuthenticationTokenIsSerializedCorrectly() throws Exception { + List<PartitionQuery<String, Employee>> partitions = + employeeStore.getPartitions(employeeStore.newQuery()); + LOG.debug("partitions {}", partitions); + } +}
