Author: alfonsonishikawa Date: Fri Mar 22 00:02:17 2013 New Revision: 1459595
URL: http://svn.apache.org/r1459595 Log: Updated with new tests for testing recursive schemas, nested schemas and three-types unions. HBase fix to pass those tests. The new tests are: DataStoreTestBase#testGetRecusive() DataStoreTestBase#testGetDoubleRecursive() DataStoreTestBase#testGetNested() DataStoreTestBase#testGet3UnionField() Modified: gora/branches/GORA_174/gora-core/src/examples/avro/employee.json gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java gora/branches/GORA_174/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java gora/branches/GORA_174/gora-hbase/src/test/conf/gora-hbase-mapping.xml gora/branches/GORA_174/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/MetricDatum.java gora/branches/GORA_174/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/Pageview.java Modified: gora/branches/GORA_174/gora-core/src/examples/avro/employee.json URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/examples/avro/employee.json?rev=1459595&r1=1459594&r2=1459595&view=diff ============================================================================== --- gora/branches/GORA_174/gora-core/src/examples/avro/employee.json (original) +++ gora/branches/GORA_174/gora-core/src/examples/avro/employee.json Fri Mar 22 00:02:17 2013 @@ -6,6 +6,28 @@ {"name": "name", "type": "string"}, {"name": "dateOfBirth", "type": "long"}, {"name": "ssn", "type": "string"}, - {"name": "salary", "type": "int"} + {"name": "salary", "type": "int"}, + {"name": "boss", "type":["null","Employee","string"]}, + {"name": "webpage", "type":["null", + { + "type": "record", + "name": "WebPage", + "namespace": "org.apache.gora.examples.generated", + "fields" : [ + {"name": "url", "type": "string"}, + {"name": "content", "type": ["null","bytes"]}, + {"name": "parsedContent", "type": {"type":"array", "items": "string"}}, + {"name": "outlinks", "type": {"type":"map", "values":"string"}}, + {"name": "metadata", "type": { + "name": "Metadata", + "type": "record", + "namespace": "org.apache.gora.examples.generated", + "fields": [ + {"name": "version", "type": "int"}, + {"name": "data", "type": {"type": "map", "values": "string"}} + ] + }} + ] + }]} ] } Modified: gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java?rev=1459595&r1=1459594&r2=1459595&view=diff ============================================================================== --- gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java (original) +++ gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java Fri Mar 22 00:02:17 2013 @@ -41,12 +41,14 @@ import org.apache.gora.persistency.ListG @SuppressWarnings("all") public class Employee extends PersistentBase { - public static final Schema _SCHEMA = Schema.parse("{\"type\":\"record\",\"name\":\"Employee\",\"namespace\":\"org.apache.gora.examples.generated\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"dateOfBirth\",\"type\":\"long\"},{\"name\":\"ssn\",\"type\":\"string\"},{\"name\":\"salary\",\"type\":\"int\"}]}"); + public static final Schema _SCHEMA = Schema.parse("{\"type\":\"record\",\"name\":\"Employee\",\"namespace\":\"org.apache.gora.examples.generated\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"dateOfBirth\",\"type\":\"long\"},{\"name\":\"ssn\",\"type\":\"string\"},{\"name\":\"salary\",\"type\":\"int\"},{\"name\":\"boss\",\"type\":[\"null\",\"Employee\",\"string\"]},{\"name\":\"webpage\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"WebPage\",\"fields\":[{\"name\":\"url\",\"type\":\"string\"},{\"name\":\"content\",\"type\":[\"null\",\"bytes\"]},{\"name\":\"parsedContent\",\"type\":{\"type\":\"array\",\"items\":\"string\"}},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"metadata\",\"type\":{\"type\":\"record\",\"name\":\"Metadata\",\"fields\":[{\"name\":\"version\",\"type\":\"int\"},{\"name\":\"data\",\"type\":{\"type\":\"map\",\"values\":\"string\"}}]}}]}]}]}"); public static enum Field { NAME(0,"name"), DATE_OF_BIRTH(1,"dateOfBirth"), SSN(2,"ssn"), SALARY(3,"salary"), + BOSS(4,"boss"), + WEBPAGE(5,"webpage"), ; private int index; private String name; @@ -55,7 +57,7 @@ public class Employee extends Persistent public String getName() {return name;} public String toString() {return name;} }; - public static final String[] _ALL_FIELDS = {"name","dateOfBirth","ssn","salary",}; + public static final String[] _ALL_FIELDS = {"name","dateOfBirth","ssn","salary","boss","webpage",}; static { PersistentBase.registerFields(Employee.class, _ALL_FIELDS); } @@ -63,6 +65,8 @@ public class Employee extends Persistent private long dateOfBirth; private Utf8 ssn; private int salary; + private Object boss; + private WebPage webpage; public Employee() { this(new StateManagerImpl()); } @@ -79,6 +83,8 @@ public class Employee extends Persistent case 1: return dateOfBirth; case 2: return ssn; case 3: return salary; + case 4: return boss; + case 5: return webpage; default: throw new AvroRuntimeException("Bad index"); } } @@ -91,6 +97,8 @@ public class Employee extends Persistent case 1:dateOfBirth = (Long)_value; break; case 2:ssn = (Utf8)_value; break; case 3:salary = (Integer)_value; break; + case 4:boss = (Object)_value; break; + case 5:webpage = (WebPage)_value; break; default: throw new AvroRuntimeException("Bad index"); } } @@ -118,4 +126,19 @@ public class Employee extends Persistent public void setSalary(int value) { put(3, value); } + public Object getBoss() { + return (Object) get(4); + } + public void setBoss(Employee value) { + put(4, value); + } + public void setBoss(Utf8 value) { + put(4, value); + } + public WebPage getWebpage() { + return (WebPage) get(5); + } + public void setWebpage(WebPage value) { + put(5, value); + } } Modified: gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java?rev=1459595&r1=1459594&r2=1459595&view=diff ============================================================================== --- gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java (original) +++ gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java Fri Mar 22 00:02:17 2013 @@ -279,6 +279,54 @@ public abstract class DataStoreTestBase } @Test + /** + * Tests put and get a record with a nested recursive record + * Employee with a boss (nested). + * @throws IOException + * @throws Exception + */ + public void testGetRecursive() throws IOException, Exception { + log.info("test method: testGetRecursive") ; + DataStoreTestUtil.testGetEmployeeRecursive(employeeStore) ; + } + + @Test + /** + * Tests put and get a record with a double nested recursive record + * Employee with a boss (nested). + * @throws IOException + * @throws Exception + */ + public void testGetDoubleRecursive() throws IOException, Exception { + log.info("test method: testGetDoubleRecursive") ; + DataStoreTestUtil.testGetEmployeeDoubleRecursive(employeeStore) ; + } + + @Test + /** + * Tests put and get a record with a nested record (not recursive) + * The webpage of an Employee + * @throws IOException + * @throws Exception + */ + public void testGetNested() throws IOException, Exception { + log.info("test method: testGetNested") ; + DataStoreTestUtil.testGetEmployeeNested(employeeStore) ; + } + + @Test + /** + * Tests put and get a record with a 3 types union, and + * having the value of the 3rd type. + * @throws IOException + * @throws Exception + */ + public void testGet3UnionField() throws IOException, Exception { + log.info("test method: testGet3UnionField") ; + DataStoreTestUtil.testGetEmployee3UnionField(employeeStore) ; + } + + @Test public void testGetWithFields() throws IOException, Exception { log.info("test method: testGetWithFields"); DataStoreTestUtil.testGetEmployeeWithFields(employeeStore); Modified: gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java?rev=1459595&r1=1459594&r2=1459595&view=diff ============================================================================== --- gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java (original) +++ gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java Fri Mar 22 00:02:17 2013 @@ -42,8 +42,11 @@ import org.apache.avro.generic.GenericAr import org.apache.avro.util.Utf8; import org.apache.gora.examples.WebPageDataCreator; import org.apache.gora.examples.generated.Employee; +import org.apache.gora.examples.generated.Metadata; import org.apache.gora.examples.generated.WebPage; +import org.apache.gora.persistency.BeanFactory; import org.apache.gora.persistency.Persistent; +import org.apache.gora.persistency.impl.BeanFactoryImpl; import org.apache.gora.query.PartitionQuery; import org.apache.gora.query.Query; import org.apache.gora.query.Result; @@ -91,6 +94,17 @@ public class DataStoreTestUtil { return employee; } + public static <K> Employee createBoss( + DataStore<K, Employee> dataStore) throws IOException, Exception { + + Employee employee = dataStore.newPersistent(); + employee.setName(new Utf8("Random boss")); + employee.setDateOfBirth( System.currentTimeMillis() - 22L * YEAR_IN_MS ); + employee.setSalary(1000000); + employee.setSsn(new Utf8("202020202020")); + return employee; + } + public static void testAutoCreateSchema(DataStore<String,Employee> dataStore) throws IOException, Exception { //should not throw exception @@ -147,6 +161,76 @@ public class DataStoreTestUtil { Assert.assertEquals(employee, after); } + + public static void testGetEmployeeRecursive(DataStore<String, Employee> dataStore) + throws IOException, Exception { + + Employee employee = DataStoreTestUtil.createEmployee(dataStore); + Employee boss = DataStoreTestUtil.createBoss(dataStore); + employee.setBoss(boss) ; + + String ssn = employee.getSsn().toString(); + dataStore.put(ssn, employee); + dataStore.flush(); + Employee after = dataStore.get(ssn, Employee._ALL_FIELDS); + Assert.assertEquals(employee, after); + Assert.assertEquals(boss, after.getBoss()) ; + } + + public static void testGetEmployeeDoubleRecursive(DataStore<String, Employee> dataStore) + throws IOException, Exception { + + Employee employee = DataStoreTestUtil.createEmployee(dataStore); + Employee boss = DataStoreTestUtil.createBoss(dataStore); + Employee uberBoss = DataStoreTestUtil.createBoss(dataStore); + uberBoss.setName(new Utf8("Ãberboss")) ; + boss.setBoss(uberBoss) ; + employee.setBoss(boss) ; + + String ssn = employee.getSsn().toString(); + dataStore.put(ssn, employee); + dataStore.flush(); + Employee after = dataStore.get(ssn, Employee._ALL_FIELDS); + Assert.assertEquals(employee, after); + Assert.assertEquals(boss, after.getBoss()) ; + Assert.assertEquals(uberBoss, ((Employee)after.getBoss()).getBoss()) ; + } + + public static void testGetEmployeeNested(DataStore<String, Employee> dataStore) + throws IOException, Exception { + + Employee employee = DataStoreTestUtil.createEmployee(dataStore); + WebPage webpage = new BeanFactoryImpl<String,WebPage>(String.class,WebPage.class).newPersistent() ; + + webpage.setUrl(new Utf8("url..")) ; + webpage.setContent(ByteBuffer.wrap("test content".getBytes())) ; + Metadata metadata = new BeanFactoryImpl<String,Metadata>(String.class,Metadata.class).newPersistent() ; + webpage.setMetadata(metadata) ; + employee.setWebpage(webpage) ; + + String ssn = employee.getSsn().toString(); + + dataStore.put(ssn, employee); + dataStore.flush(); + Employee after = dataStore.get(ssn, Employee._ALL_FIELDS); + Assert.assertEquals(employee, after); + Assert.assertEquals(webpage, after.getWebpage()) ; + } + + public static void testGetEmployee3UnionField(DataStore<String, Employee> dataStore) + throws IOException, Exception { + + Employee employee = DataStoreTestUtil.createEmployee(dataStore); + employee.setBoss(new Utf8("Real boss")) ; + + String ssn = employee.getSsn().toString(); + dataStore.put(ssn, employee); + dataStore.flush(); + Employee after = dataStore.get(ssn, Employee._ALL_FIELDS); + Assert.assertEquals(employee, after); + Assert.assertEquals("Real boss", ((Utf8)after.getBoss()).toString()) ; + } + public static void testGetEmployeeNonExisting(DataStore<String, Employee> dataStore) throws IOException, Exception { Employee employee = dataStore.get("_NON_EXISTING_SSN_FOR_EMPLOYEE_"); Modified: gora/branches/GORA_174/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java?rev=1459595&r1=1459594&r2=1459595&view=diff ============================================================================== --- gora/branches/GORA_174/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java (original) +++ gora/branches/GORA_174/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java Fri Mar 22 00:02:17 2013 @@ -190,9 +190,14 @@ implements Configurable { try{ fields = getFieldsToQuery(fields); Get get = new Get(toBytes(key)); - addFields(get, fields); - Result result = table.get(get); - return newInstance(result, fields); + + if (table.exists(get)) { + addFields(get, fields); + Result result = table.get(get); + return newInstance(result, fields); + } else { + return null ; + } } catch(IOException ex2){ LOG.error(ex2.getMessage()); LOG.error(ex2.getStackTrace().toString()); @@ -511,9 +516,18 @@ implements Configurable { } @SuppressWarnings({ "unchecked", "rawtypes" }) + /** + * Creates a new Persistent instance with the values in 'result' for the fields listed. + * @param result result form a HTable#get() + * @param fields List of fields queried, or null for all + * @return A new instance with default values for not listed fields + * null if 'result' is null. + * @throws IOException + */ public T newInstance(Result result, String[] fields) throws IOException { - if(result == null || result.isEmpty()) + // TODO Set the default value for not listed fields. + if(result == null) return null; T persistent = newPersistent(); Modified: gora/branches/GORA_174/gora-hbase/src/test/conf/gora-hbase-mapping.xml URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-hbase/src/test/conf/gora-hbase-mapping.xml?rev=1459595&r1=1459594&r2=1459595&view=diff ============================================================================== --- gora/branches/GORA_174/gora-hbase/src/test/conf/gora-hbase-mapping.xml (original) +++ gora/branches/GORA_174/gora-hbase/src/test/conf/gora-hbase-mapping.xml Fri Mar 22 00:02:17 2013 @@ -35,6 +35,8 @@ <field name="dateOfBirth" family="info" qualifier="db"/> <field name="ssn" family="info" qualifier="sn"/> <field name="salary" family="info" qualifier="sl"/> + <field name="boss" family="info" qualifier="bs"/> + <field name="webpage" family="info" qualifier="wp"/> </class> <class name="org.apache.gora.examples.generated.WebPage" keyClass="java.lang.String" table="WebPage"> Modified: gora/branches/GORA_174/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/MetricDatum.java URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/MetricDatum.java?rev=1459595&r1=1459594&r2=1459595&view=diff ============================================================================== --- gora/branches/GORA_174/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/MetricDatum.java (original) +++ gora/branches/GORA_174/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/MetricDatum.java Fri Mar 22 00:02:17 2013 @@ -1,28 +1,43 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + *Licensed to the Apache Software Foundation (ASF) under one + *or more contributor license agreements. See the NOTICE file + *distributed with this work for additional information + *regarding copyright ownership. The ASF licenses this file + *to you under the Apache License, Version 2.0 (the" + *License"); you may not use this file except in compliance + *with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + *Unless required by applicable law or agreed to in writing, software + *distributed under the License is distributed on an "AS IS" BASIS, + *WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + *See the License for the specific language governing permissions and + *limitations under the License. */ + package org.apache.gora.tutorial.log.generated; -import org.apache.avro.AvroRuntimeException; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.HashMap; +import org.apache.avro.Protocol; import org.apache.avro.Schema; +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.Protocol; import org.apache.avro.util.Utf8; +import org.apache.avro.ipc.AvroRemoteException; +import org.apache.avro.generic.GenericArray; +import org.apache.avro.specific.FixedSize; +import org.apache.avro.specific.SpecificExceptionBase; +import org.apache.avro.specific.SpecificRecordBase; +import org.apache.avro.specific.SpecificRecord; +import org.apache.avro.specific.SpecificFixed; import org.apache.gora.persistency.StateManager; import org.apache.gora.persistency.impl.PersistentBase; import org.apache.gora.persistency.impl.StateManagerImpl; +import org.apache.gora.persistency.StatefulHashMap; +import org.apache.gora.persistency.ListGenericArray; @SuppressWarnings("all") public class MetricDatum extends PersistentBase { Modified: gora/branches/GORA_174/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/Pageview.java URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/Pageview.java?rev=1459595&r1=1459594&r2=1459595&view=diff ============================================================================== --- gora/branches/GORA_174/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/Pageview.java (original) +++ gora/branches/GORA_174/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/Pageview.java Fri Mar 22 00:02:17 2013 @@ -1,28 +1,43 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + *Licensed to the Apache Software Foundation (ASF) under one + *or more contributor license agreements. See the NOTICE file + *distributed with this work for additional information + *regarding copyright ownership. The ASF licenses this file + *to you under the Apache License, Version 2.0 (the" + *License"); you may not use this file except in compliance + *with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + *Unless required by applicable law or agreed to in writing, software + *distributed under the License is distributed on an "AS IS" BASIS, + *WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + *See the License for the specific language governing permissions and + *limitations under the License. */ + package org.apache.gora.tutorial.log.generated; -import org.apache.avro.AvroRuntimeException; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.HashMap; +import org.apache.avro.Protocol; import org.apache.avro.Schema; +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.Protocol; import org.apache.avro.util.Utf8; +import org.apache.avro.ipc.AvroRemoteException; +import org.apache.avro.generic.GenericArray; +import org.apache.avro.specific.FixedSize; +import org.apache.avro.specific.SpecificExceptionBase; +import org.apache.avro.specific.SpecificRecordBase; +import org.apache.avro.specific.SpecificRecord; +import org.apache.avro.specific.SpecificFixed; import org.apache.gora.persistency.StateManager; import org.apache.gora.persistency.impl.PersistentBase; import org.apache.gora.persistency.impl.StateManagerImpl; +import org.apache.gora.persistency.StatefulHashMap; +import org.apache.gora.persistency.ListGenericArray; @SuppressWarnings("all") public class Pageview extends PersistentBase {
