Added: hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestMultiplePartitions.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestMultiplePartitions.java?rev=934649&view=auto ============================================================================== --- hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestMultiplePartitions.java (added) +++ hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestMultiplePartitions.java Thu Apr 15 23:56:44 2010 @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.owl.driver; + + +import java.util.ArrayList; +import java.util.List; + +import junit.framework.Assert; + +import org.apache.hadoop.owl.OwlTestCase; +import org.apache.hadoop.owl.common.OwlException; +import org.apache.hadoop.owl.protocol.ColumnType; +import org.apache.hadoop.owl.protocol.OwlColumnSchema; +import org.apache.hadoop.owl.protocol.OwlDatabase; +import org.apache.hadoop.owl.protocol.OwlKeyValue; +import org.apache.hadoop.owl.protocol.OwlLoaderInfo; +import org.apache.hadoop.owl.protocol.OwlPartition; +import org.apache.hadoop.owl.protocol.OwlSchema; +import org.apache.hadoop.owl.protocol.OwlTable; +import org.apache.hadoop.owl.protocol.OwlTableName; +import org.apache.hadoop.owl.protocol.OwlKey.DataType; +import org.apache.hadoop.owl.schema.ParseException; +import org.apache.hadoop.owl.schema.Schema; +import org.junit.Test; + +public class TestMultiplePartitions extends OwlTestCase{ + + private static String dbname = "testmultiplepartitiondb"; + private static String tableName = "testmultiplepartitiontable"; + + private static OwlTableName name = new OwlTableName(dbname, tableName); + private static OwlDriver driver; + + static List<OwlKeyValue> keyValues; + static OwlKeyValue kvps1, kvps2, kvps3; + static int count = 1; + + static OwlSchema tableSchema, partSchemaP1, partSchemaP2, partSchemaP3; + + private static boolean initialized = false; + private static int numTestsRemaining = 2; + + public TestMultiplePartitions() { + } + + private static OwlLoaderInfo _instantiateOwlLoaderInfo(int i) throws OwlException{ + if ((i % 2)==0){ + return new OwlLoaderInfo("test"); + }else{ + return new OwlLoaderInfo("test",Integer.toString(i)); + } + } + + @SuppressWarnings("boxing") + static void publish(int v1, String v2, int v3, OwlSchema inputSchema) throws OwlException { + kvps1.setIntValue(v1); + kvps2.setStringValue(v2); + kvps3.setIntValue(v3); + + driver.publish(name, keyValues, null, "hdfs://dummytestlocation/1234" + count, inputSchema, _instantiateOwlLoaderInfo(v1)); + count++; + } + + public static void initialize() throws OwlException { + if (!initialized){ + driver = new OwlDriver(getUri()); + + DriverVerificationUtil.dropOwlTableIfExists(driver, name); + DriverVerificationUtil.dropOwlDatabaseIfExists(driver, dbname); + + driver.createOwlDatabase(new OwlDatabase(dbname, "hdfs://localhost:9000/data/testdatabase")); + + // table schema + OwlSchema tableSchema = new OwlSchema(); + tableSchema.addColumnSchema(new OwlColumnSchema("NAME", ColumnType.STRING)); + tableSchema.addColumnSchema(new OwlColumnSchema("SEX", ColumnType.STRING)); + tableSchema.addColumnSchema(new OwlColumnSchema("AGE", ColumnType.INT)); + + // create owl table + OwlTable table = new OwlTableBuilder() + .setName(name) + .addPartition("part1", DataType.INT) + .addPartition("part2", DataType.STRING) + .addPartition("part3", DataType.INT) + .setSchema(tableSchema) + .build(); + + driver.createOwlTable(table); + + + // partitionSchema for part1 + partSchemaP1 = new OwlSchema(); + partSchemaP1.addColumnSchema(new OwlColumnSchema("NAME", ColumnType.STRING)); + partSchemaP1.addColumnSchema(new OwlColumnSchema("SEX", ColumnType.STRING)); + partSchemaP1.addColumnSchema(new OwlColumnSchema("AGE", ColumnType.INT)); + partSchemaP1.addColumnSchema(new OwlColumnSchema("SCORE", ColumnType.FLOAT)); + + // subSubschemas + OwlSchema subSubSchema = new OwlSchema(); + subSubSchema.addColumnSchema(new OwlColumnSchema("f1", ColumnType.INT)); + subSubSchema.addColumnSchema(new OwlColumnSchema("f2", ColumnType.STRING)); + + // subSchema + OwlSchema subSchema = new OwlSchema(); + subSchema.addColumnSchema(new OwlColumnSchema("r1", ColumnType.RECORD,subSubSchema)); + + // partitionSchema for part2 + partSchemaP2 = new OwlSchema(); + partSchemaP2.addColumnSchema(new OwlColumnSchema("c10", ColumnType.COLLECTION, subSchema)); + partSchemaP2.addColumnSchema(new OwlColumnSchema("c1", ColumnType.INT)); + + // partitionSchema for part3 + partSchemaP3 = new OwlSchema(); + partSchemaP3.addColumnSchema(new OwlColumnSchema("c3", ColumnType.INT)); + partSchemaP3.addColumnSchema(new OwlColumnSchema("c2", ColumnType.STRING)); + partSchemaP3.addColumnSchema(new OwlColumnSchema("c1", ColumnType.INT)); + + + // publish partitions + // set the key for partition key value pairs here + keyValues = new ArrayList<OwlKeyValue>(); + kvps1 = new OwlKeyValue("part1", 1); + kvps2 = new OwlKeyValue("part2", "aaa"); + kvps3 = new OwlKeyValue("part3", 3); + + keyValues.add(kvps1); + keyValues.add(kvps2); + keyValues.add(kvps3); + + publish(1, "aaa", 3, partSchemaP1); + publish(2, "aaa", 5, partSchemaP2); + publish(3, "bbb", 4, partSchemaP3); + initialized = true; + } + } + + @Test + public static void testRunSelectOwlTable() throws OwlException { + initialize(); + OwlTable owlTable = driver.getOwlTable(name); + OwlSchema schema = owlTable.getSchema(); + System.out.println("owltable Schema is " + schema.getSchemaString()); + + Assert.assertTrue + ( + "part1:int,part2:string,part3:int,name:string,sex:string,age:int,score:float,c10:collection(r1:record(f1:int,f2:string)),c1:int,c3:int,c2:string".equalsIgnoreCase( + schema.getSchemaString() ) + ); + cleanup(); + } + + public static void runSelectLeafPartition(String filter, int expectedCount, String expectedSchema) throws OwlException { + System.out.println("Filter : " + filter + " ExpectedCount : " + expectedCount); + + List<OwlPartition> partitions = driver.getPartitions(name, filter); + assertNotNull(partitions); + assertEquals(expectedCount, partitions.size()); + + for (OwlPartition ptn : partitions){ + OwlLoaderInfo loaderInfo = ptn.getLoader(); + + // make sure the partition is a leaf level one + Assert.assertTrue (ptn.isLeaf()); + // make sure the schemaString is not null + Assert.assertTrue(ptn.getSchema().getSchemaString() != null); + + System.out.println("[orange]partition level schema is " + ptn.getSchema().getSchemaString() ); + Assert.assertEquals(expectedSchema, ptn.getSchema().getSchemaString()); + + // verify loader info. + assertNotNull(loaderInfo); + for (OwlKeyValue kv:ptn.getKeyValues()){ + if (kv.getKeyName().equalsIgnoreCase("part1")){ + _verifyPartitionOwlLoaderInfo(loaderInfo, kv); + } + } + } + } + + private static void _verifyPartitionOwlLoaderInfo(OwlLoaderInfo loaderInfo, + OwlKeyValue kv) { + int i = kv.getIntValue().intValue(); + if ((i % 2) == 0){ + assertNull(loaderInfo.getInputDriverArgs()); + }else { + assertNotNull(loaderInfo.getInputDriverArgs()); + assertEquals(kv.getIntValue().toString(),loaderInfo.getInputDriverArgs()); + } + } + + @Test + public static void testSelectLeafPartitionLevel() throws OwlException { + initialize(); + runSelectLeafPartition("part1 = 1", 1, partSchemaP1.getSchemaString()); + runSelectLeafPartition("part1 = 2", 1, partSchemaP2.getSchemaString()); + runSelectLeafPartition("part1 = 3", 1, partSchemaP3.getSchemaString()); + cleanup(); + } + + public static void cleanup() throws OwlException { + numTestsRemaining--; + if (numTestsRemaining == 0){ + initialize(); + driver.dropOwlTable(new OwlTable(name)); + driver.dropOwlDatabase(new OwlDatabase(dbname, null)); + } + } +}
Added: hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestNestedSchema.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestNestedSchema.java?rev=934649&view=auto ============================================================================== --- hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestNestedSchema.java (added) +++ hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestNestedSchema.java Thu Apr 15 23:56:44 2010 @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.owl.driver; + +import org.apache.hadoop.owl.OwlTestCase; +import org.apache.hadoop.owl.common.ErrorType; +import org.apache.hadoop.owl.common.OwlException; +import org.apache.hadoop.owl.protocol.ColumnType; +import org.apache.hadoop.owl.protocol.OwlColumnSchema; +import org.apache.hadoop.owl.protocol.OwlDatabase; +import org.apache.hadoop.owl.protocol.OwlSchema; +import org.apache.hadoop.owl.protocol.OwlTable; +import org.apache.hadoop.owl.protocol.OwlTableName; +import org.apache.hadoop.owl.protocol.OwlKey.DataType; +import org.apache.hadoop.owl.testdriver.CompareOwlObjects; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestNestedSchema { + + private static String dbname = "testDriverNestedSchemaDB"; + private static String tableName = "testDriverNestedSchemaTab"; + + private static OwlTableName name = new OwlTableName(dbname, tableName); + private static OwlDriver driver; + + //MAP types cannot be used in this test since server currently adds a default schema for MAP causing compare to fail + + private static void runSchemaTest(OwlSchema inputSchema) throws Exception { + + OwlTable table = new OwlTableBuilder(). + setName(name). + addPartition("part1", DataType.INT). + setSchema(inputSchema). + build(); + + driver.createOwlTable(table); + + OwlTable outputTable = driver.getOwlTable(name); + OwlSchema outputSchema = outputTable.getSchema(); + + CompareOwlObjects.compareObjects(outputSchema, inputSchema); + + driver.dropOwlTable(outputTable); + } + + @BeforeClass + public static void setUp() throws OwlException { + driver = new OwlDriver(OwlTestCase.getUri()); + + DriverVerificationUtil.dropOwlTableIfExists(driver,name); + DriverVerificationUtil.dropOwlDatabaseIfExists(driver,dbname); + + driver.createOwlDatabase(new OwlDatabase(dbname, "hdfs://localhost:9000/data/testdatabase")); + } + + @Test + public void testNestedScehma1() throws Exception { + OwlSchema schema = new OwlSchema(); + schema.addColumnSchema(new OwlColumnSchema("part1", ColumnType.INT)); + schema.addColumnSchema(new OwlColumnSchema("c1", ColumnType.INT)); + + runSchemaTest(schema); + } + + @Test + public void testNestedScehma2() throws Exception { + OwlSchema schema = new OwlSchema(); + schema.addColumnSchema(new OwlColumnSchema("part1", ColumnType.INT)); + schema.addColumnSchema(new OwlColumnSchema("c1", ColumnType.INT)); + schema.addColumnSchema(new OwlColumnSchema("c2", ColumnType.BYTES)); + + runSchemaTest(schema); + } + + @Test + public void testNestedScehma3() throws Exception { + OwlSchema subSubSchema = new OwlSchema(); + subSubSchema.addColumnSchema(new OwlColumnSchema("c11", ColumnType.INT)); + + OwlSchema subSchema = new OwlSchema(); + subSchema.addColumnSchema(new OwlColumnSchema("r1", ColumnType.RECORD, subSubSchema)); + + OwlSchema schema = new OwlSchema(); + schema.addColumnSchema(new OwlColumnSchema("part1", ColumnType.INT)); + schema.addColumnSchema(new OwlColumnSchema("c1", ColumnType.COLLECTION, subSchema)); + + runSchemaTest(schema); + } + + @Test + public void testNestedScehma4() throws Exception { + OwlSchema subSchema1 = new OwlSchema(); + subSchema1.addColumnSchema(new OwlColumnSchema("c111", ColumnType.BYTES)); + + OwlSchema subSchema2 = new OwlSchema(); + subSchema2.addColumnSchema(new OwlColumnSchema("c11", ColumnType.RECORD, subSchema1)); + + OwlSchema schema = new OwlSchema(); + schema.addColumnSchema(new OwlColumnSchema("part1", ColumnType.INT)); + schema.addColumnSchema(new OwlColumnSchema("c1", ColumnType.COLLECTION, subSchema2)); + schema.addColumnSchema(new OwlColumnSchema("c2", ColumnType.INT)); + + runSchemaTest(schema); + } + + @Test + public void testRecordError() throws Exception { + OwlSchema schema = new OwlSchema(); + schema.addColumnSchema(new OwlColumnSchema("c1", ColumnType.RECORD)); + + OwlException exc = null; + try { + runSchemaTest(schema); + }catch(OwlException e) { + exc = e; + } + + Assert.assertNotNull(exc); + Assert.assertEquals(ErrorType.ERROR_MISSING_SUBSCHEMA, exc.getErrorType()); + } + + @Test + public void testCollectionError() throws Exception { + OwlSchema schema = new OwlSchema(); + schema.addColumnSchema(new OwlColumnSchema("c1", ColumnType.COLLECTION)); + + OwlException exc = null; + try { + runSchemaTest(schema); + }catch(OwlException e) { + exc = e; + } + + Assert.assertNotNull(exc); + Assert.assertEquals(ErrorType.ERROR_MISSING_SUBSCHEMA, exc.getErrorType()); + } + + @Test + public void testSchemaAlter() throws Exception { + + OwlSchema schema = new OwlSchema(); + schema.addColumnSchema(new OwlColumnSchema("part1", ColumnType.INT)); + schema.addColumnSchema(new OwlColumnSchema("c1", ColumnType.INT)); + + OwlTable table = new OwlTableBuilder(). + setName(name). + addPartition("part1", DataType.INT). + setSchema(schema). + build(); + + driver.createOwlTable(table); + + OwlSchema newSchema = new OwlSchema(); + newSchema.addColumnSchema(new OwlColumnSchema("t1", ColumnType.INT)); + newSchema.addColumnSchema(new OwlColumnSchema("part1", ColumnType.INT)); + + driver.alterOwlTable(AlterOwlTableCommand.createForSetSchema(name, newSchema)); + + OwlTable outputTable = driver.getOwlTable(name); + OwlSchema outputSchema = outputTable.getSchema(); + CompareOwlObjects.compareObjects(outputSchema, newSchema); + + driver.dropOwlTable(outputTable); + } + + @AfterClass + public static void tearDown() throws OwlException { + driver.dropOwlDatabase(new OwlDatabase(dbname, null)); + } +} Added: hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestNonPartitionedTable.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestNonPartitionedTable.java?rev=934649&view=auto ============================================================================== --- hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestNonPartitionedTable.java (added) +++ hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestNonPartitionedTable.java Thu Apr 15 23:56:44 2010 @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.owl.driver; + +import java.util.List; + +import org.apache.hadoop.owl.OwlTestCase; +import org.apache.hadoop.owl.common.ErrorType; +import org.apache.hadoop.owl.common.OwlException; +import org.apache.hadoop.owl.protocol.ColumnType; +import org.apache.hadoop.owl.protocol.OwlColumnSchema; +import org.apache.hadoop.owl.protocol.OwlDatabase; +import org.apache.hadoop.owl.protocol.OwlLoaderInfo; +import org.apache.hadoop.owl.protocol.OwlPartition; +import org.apache.hadoop.owl.protocol.OwlSchema; +import org.apache.hadoop.owl.protocol.OwlTable; +import org.apache.hadoop.owl.protocol.OwlTableName; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestNonPartitionedTable { + + private static String dbname = "testDnonpartitiondb"; + private static String tableName = "testDnonpartitiontab"; + + private static OwlTableName name = new OwlTableName(dbname, tableName); + private static OwlDriver driver; + + private static OwlSchema schema = null; + + @BeforeClass + public static void setUp() throws OwlException { + driver = new OwlDriver(OwlTestCase.getUri()); + + DriverVerificationUtil.dropOwlTableIfExists(driver,name); + DriverVerificationUtil.dropOwlDatabaseIfExists(driver,dbname); + + driver.createOwlDatabase(new OwlDatabase(dbname, "hdfs://localhost:9000/data/testNPdatabase")); + + schema = new OwlSchema(); + schema.addColumnSchema(new OwlColumnSchema("c1", ColumnType.INT)); + + OwlTable table = new OwlTableBuilder(). + setName(name). + setSchema(schema). + build(); + + driver.createOwlTable(table); + + driver.publish(name, null, null, "partNPloc", schema, new OwlLoaderInfo("test")); + } + + @AfterClass + public static void tearDown() throws OwlException { + driver.dropOwlTable(new OwlTable(name)); + driver.dropOwlDatabase(new OwlDatabase(dbname, null)); + } + + @Test + public void testSelectPartition() throws Exception { + List<OwlPartition> pList = driver.getPartitions(name, null); + Assert.assertEquals(1, pList.size()); + Assert.assertEquals(pList.get(0).getStorageLocation(), "partNPloc"); + } + + @Test + public void testDropPartition() throws Exception { + driver.dropPartition(name, null); + + List<OwlPartition> pList = driver.getPartitions(name, null); + Assert.assertEquals(0, pList.size()); + + driver.publish(name, null, null, "partNPloc", schema, new OwlLoaderInfo("test")); + } + + @Test + public void testDuplicatePublish() throws Exception { + OwlException exc = null; + try { + OwlSchema schema = new OwlSchema(); + schema.addColumnSchema(new OwlColumnSchema("c1", ColumnType.INT)); + + driver.publish(name, null, null, "partNPloc2", schema, new OwlLoaderInfo("test")); + } catch(OwlException e ) { + exc = e; + } + + Assert.assertNotNull(exc); + Assert.assertEquals(ErrorType.ERROR_DUPLICATE_PUBLISH, exc.getErrorType()); + } + + @SuppressWarnings("boxing") + @Test + public void testDescribe() throws OwlException { + OwlTable table = driver.getOwlTable(name); + Assert.assertEquals(name, table.getName()); + Assert.assertEquals(0, table.getPartitionKeys().size()); + } +} Added: hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestOwlSchema.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestOwlSchema.java?rev=934649&view=auto ============================================================================== --- hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestOwlSchema.java (added) +++ hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestOwlSchema.java Thu Apr 15 23:56:44 2010 @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.owl.driver; + + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.owl.OwlTestCase; +import org.apache.hadoop.owl.common.ErrorType; +import org.apache.hadoop.owl.common.OwlException; +import org.apache.hadoop.owl.common.OwlUtil; +import org.apache.hadoop.owl.protocol.ColumnType; +import org.apache.hadoop.owl.protocol.OwlColumnSchema; +import org.apache.hadoop.owl.protocol.OwlDatabase; +import org.apache.hadoop.owl.protocol.OwlKeyValue; +import org.apache.hadoop.owl.protocol.OwlLoaderInfo; +import org.apache.hadoop.owl.protocol.OwlPartition; +import org.apache.hadoop.owl.protocol.OwlSchema; +import org.apache.hadoop.owl.protocol.OwlTable; +import org.apache.hadoop.owl.protocol.OwlTableName; +import org.apache.hadoop.owl.protocol.OwlKey.DataType; +import org.junit.Test; + +public class TestOwlSchema extends OwlTestCase{ + + private static String dbname = "testowlschemadb"; + private static String tableName = "testowlschematab"; + + private static OwlTableName name = new OwlTableName(dbname, tableName); + private static OwlDriver driver; + + static List<OwlKeyValue> keyValues; + static OwlKeyValue k1, k2, k3; + static int count = 1; + + static OwlSchema partitionSchema1 = new OwlSchema(); + static OwlSchema partitionSchema2 = new OwlSchema(); + static OwlSchema partitionSchema3 = new OwlSchema(); + + private static boolean initialized = false; + private static int numTestsRemaining = 1; + + public TestOwlSchema() { + } + + private static OwlLoaderInfo _instantiateOwlLoaderInfo(int i) throws OwlException{ + if ((i % 2)==0){ + return new OwlLoaderInfo("test"); + }else{ + return new OwlLoaderInfo("test",Integer.toString(i)); + } + } + + @SuppressWarnings("boxing") + static void publish(int v1, String v2, int v3, OwlSchema inputSchema) throws OwlException { + k1.setIntValue(v1); + k2.setStringValue(v2); + k3.setIntValue(v3); + // TODO if we change the deLocation to "testselectloc" + count, data nucleus would throw an exception, + // "JDOQL Single-String query has a subquery clause without a closing parenthesis". -- we think this is a bug in data nucleus + driver.publish(name, keyValues, null, "hdfs://dummylocation/1234" + count, inputSchema, _instantiateOwlLoaderInfo(v1)); + count++; + } + + public static void initialize() throws OwlException { + if (!initialized){ + driver = new OwlDriver(getUri()); + + DriverVerificationUtil.dropOwlTableIfExists(driver, name); + DriverVerificationUtil.dropOwlDatabaseIfExists(driver, dbname); + + driver.createOwlDatabase(new OwlDatabase(dbname, "hdfs://localhost:9000/data/testdatabase")); + + OwlSchema tableSchema = new OwlSchema(); + tableSchema.addColumnSchema(new OwlColumnSchema("date", ColumnType.LONG)); + + OwlTable table = new OwlTableBuilder(). + setName(name). + addPartition("part1", DataType.INT). + addPartition("part2", DataType.STRING). + addPartition("part3", DataType.INT). + setSchema(tableSchema). + build(); + + driver.createOwlTable(table); + + keyValues = new ArrayList<OwlKeyValue>(); + k1 = new OwlKeyValue("part1", 1); + k2 = new OwlKeyValue("part2", "aaa"); + k3 = new OwlKeyValue("part3", 3); + keyValues.add(k1); + keyValues.add(k2); + keyValues.add(k3); + + // test schema -- c1:collection(r1:record(i1:int)) + OwlSchema subSubSchema = new OwlSchema(); + subSubSchema.addColumnSchema(new OwlColumnSchema("i1", ColumnType.INT)); + OwlSchema subSchema = new OwlSchema(); + subSchema.addColumnSchema(new OwlColumnSchema("r1", ColumnType.RECORD, subSubSchema)); + partitionSchema1.addColumnSchema(new OwlColumnSchema("c1", ColumnType.COLLECTION, subSchema)); + + publish(1, "aaa", 3, partitionSchema1); + + // test schema -- c2:collection(record(i1:long)) consolidation with partitionSchema1 should succeed + OwlSchema subSubSchema2 = new OwlSchema(); + subSubSchema2.addColumnSchema(new OwlColumnSchema("i1", ColumnType.LONG)); + OwlSchema subSchema2 = new OwlSchema(); + subSchema2.addColumnSchema(new OwlColumnSchema(ColumnType.RECORD, subSubSchema2)); + partitionSchema2.addColumnSchema(new OwlColumnSchema("c2", ColumnType.COLLECTION, subSchema2)); + publish(1, "bbb", 3, partitionSchema2); + + // test schema -- c1:collection(record(i1:int )) consolidation with partitionSchema1 consolidation with partitionSchema1 should *fail* + OwlSchema subSubSchema3 = new OwlSchema(); + subSubSchema3.addColumnSchema(new OwlColumnSchema("i1", ColumnType.INT)); + OwlSchema subSchema3 = new OwlSchema(); + subSchema3.addColumnSchema(new OwlColumnSchema(ColumnType.RECORD, subSubSchema3)); + partitionSchema3.addColumnSchema(new OwlColumnSchema("c1", ColumnType.COLLECTION, subSchema3)); + + try{ + publish(1, "bbb", 4, partitionSchema3); + }catch(OwlException e){ + if (e.getErrorCode() == ErrorType.ZEBRA_SCHEMA_EXCEPTION.getErrorCode()){ + // good. this is a negative test case + System.out.println("ErrorType is : " + e.getErrorCode()); + }else{ + throw e; + } + } + // publish(1, "bbb", 4); + // publish(2, "aaa", 5); + // publish(2, "bbb", 4); + + initialized = true; + } + } + + public static void runSelect(String filter, String partitionKeyName, OwlSchema expectedSchema) throws OwlException { + System.out.println("Filter : " + filter + " Key : " + partitionKeyName + " ExpectedSchema : " + expectedSchema.getSchemaString()); + + List<OwlPartition> partitions = driver.getPartitions(name, filter, partitionKeyName); + assertNotNull(partitions); + + for (OwlPartition ptn : partitions){ + if (ptn.isLeaf()){ + // only leaf level partition has data schema + assertEquals(expectedSchema.getSchemaString(), (ptn.getSchema()).getSchemaString()); + }else{ + assertFalse("selected schema is not leaf. only leaf schema can have data schema", false); + } + } + } + + @Test + public static void testSelectLeaf() throws OwlException { + + initialize(); + runSelect("(part1 = 1) and (part2=\"aaa\") and (part3 = 3)", null, partitionSchema1); + runSelect("(part1 = 1) and (part2=\"bbb\") and (part3 = 3)", null, partitionSchema2); + + cleanup(); + } + + public static void cleanup() throws OwlException { + numTestsRemaining--; + if (numTestsRemaining == 0){ + initialize(); + driver.dropOwlTable(new OwlTable(name)); + driver.dropOwlDatabase(new OwlDatabase(dbname, null)); + } + } +} Added: hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestOwlTable.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestOwlTable.java?rev=934649&view=auto ============================================================================== --- hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestOwlTable.java (added) +++ hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestOwlTable.java Thu Apr 15 23:56:44 2010 @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.owl.driver; + +import java.util.List; + +import org.apache.hadoop.owl.OwlTestCase; +import org.apache.hadoop.owl.common.ErrorType; +import org.apache.hadoop.owl.common.OwlException; +import org.apache.hadoop.owl.common.OwlUtil; +import org.apache.hadoop.owl.common.TestOwlUtil; +import org.apache.hadoop.owl.protocol.ColumnType; +import org.apache.hadoop.owl.protocol.OwlColumnSchema; +import org.apache.hadoop.owl.protocol.OwlDatabase; +import org.apache.hadoop.owl.protocol.OwlSchema; +import org.apache.hadoop.owl.protocol.OwlTable; +import org.apache.hadoop.owl.protocol.OwlTableName; +import org.apache.hadoop.owl.protocol.OwlKey.DataType; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestOwlTable { + + private static String dbname = "testdowltabledb"; + private static String tableName = "testdowltabletab"; + private static OwlTableName name = new OwlTableName(dbname, tableName); + private static OwlTable table = null; + + private static OwlDriver driver; + + public TestOwlTable() { + } + + @BeforeClass + public static void setUp() throws OwlException { + driver = new OwlDriver(OwlTestCase.getUri()); + DriverVerificationUtil.dropOwlTableIfExists(driver, name); + DriverVerificationUtil.dropOwlDatabaseIfExists(driver, dbname); + driver.createOwlDatabase(new OwlDatabase(dbname, "hdfs://localhost:9000/data/testdowltabledb")); + + OwlSchema subSubSchema = new OwlSchema(); + subSubSchema.addColumnSchema(new OwlColumnSchema("i1", ColumnType.INT)); + OwlSchema subSchema = new OwlSchema(); + subSchema.addColumnSchema(new OwlColumnSchema(ColumnType.RECORD, subSubSchema)); + OwlSchema schema = new OwlSchema(); + schema.addColumnSchema(new OwlColumnSchema("c1", ColumnType.COLLECTION, subSchema)); + + + table = new OwlTableBuilder(). + setName(name). + addPartition("part1", DataType.INT). + setSchema(schema). + build(); + System.out.println("Table level schema is ["+schema.getSchemaString()+"]"); + driver.createOwlTable(table); + } + + @Test + public void testOwltableName() throws OwlException { + // negative test to validate the length of owltable name is not longer than 64 characters + String owltablename_negativecase = TestOwlUtil.generateLongString(OwlUtil.IDENTIFIER_LIMIT +1); + OwlTableName name_negativecase = new OwlTableName(dbname, owltablename_negativecase); + OwlSchema schema = new OwlSchema(); + schema.addColumnSchema(new OwlColumnSchema("i1", ColumnType.STRING)); + + table = new OwlTableBuilder(). + setName(name_negativecase). + addPartition("part1", DataType.INT). + setSchema(schema). + build(); + + try{ + driver.createOwlTable(table); + Assert.fail("There is no OwlException thrown. We are expecting owltable name variable length validation fails"); + }catch(OwlException e){ + Assert.assertEquals(e.getErrorType(), ErrorType.ERROR_IDENTIFIER_LENGTH_VALIDATION_FAILED); + } + } + + @Test + public void testPartitionKeyName() throws OwlException { + // negative test to validate the length of partition key name is not longer than 64 characters + String partitionkeyname_negativecase = TestOwlUtil.generateLongString(OwlUtil.IDENTIFIER_LIMIT +1); + OwlTableName name = new OwlTableName(dbname, "testpartitionkeyname_owltable"); + OwlSchema schema = new OwlSchema(); + schema.addColumnSchema(new OwlColumnSchema("i1", ColumnType.INT)); + + table = new OwlTableBuilder(). + setName(name). + addPartition(partitionkeyname_negativecase, DataType.INT). + setSchema(schema). + build(); + + try{ + driver.createOwlTable(table); + Assert.fail("There is no OwlException thrown. We are expecting partition key length validation fails"); + }catch(OwlException e){ + Assert.assertEquals(e.getErrorType(), ErrorType.ERROR_IDENTIFIER_LENGTH_VALIDATION_FAILED); + } + } + + @Test + public void testPropertyKeyName() throws OwlException { + // negative test to validate the length of property key name is not longer than its limit + String propertykeyname_negativecase = TestOwlUtil.generateLongString(OwlUtil.IDENTIFIER_LIMIT +1); + OwlTableName name = new OwlTableName(dbname, "testpropertykeyname_owltable"); + OwlSchema schema = new OwlSchema(); + schema.addColumnSchema(new OwlColumnSchema("i1", ColumnType.INT)); + + table = new OwlTableBuilder(). + setName(name). + addPartition("part1", DataType.INT). + addProperty(propertykeyname_negativecase, DataType.STRING). + setSchema(schema). + build(); + + try{ + driver.createOwlTable(table); + Assert.fail("There is no OwlException thrown. We are expecting property key length validation fails"); + }catch(OwlException e){ + Assert.assertEquals(e.getErrorType(), ErrorType.ERROR_IDENTIFIER_LENGTH_VALIDATION_FAILED); + } + } + + @Test + public void testDescribe() throws OwlException { + OwlTable tab = driver.getOwlTable(name); + Assert.assertEquals(name, tab.getName()); + } + + @Test + public void testSelect() throws OwlException { + List<OwlTableName> allTables = driver.showOwlTables(dbname); + Assert.assertTrue(allTables.contains(name)); + } + + @SuppressWarnings("null") + @Test + public void testInvalidCreate() throws OwlException { + OwlException exc = null; + try { + driver.createOwlTable(null); + } catch(OwlException e) { + exc = e; + } + + Assert.assertNotNull(exc); + Assert.assertEquals(ErrorType.INVALID_FIELD_VALUE, exc.getErrorType()); + } + + + @SuppressWarnings("null") + @Test + public void testInvalidGet1() throws OwlException { + OwlException exc = null; + try { + OwlTable tab = driver.getOwlTable(null); + } catch(OwlException e) { + exc = e; + } + + Assert.assertNotNull(exc); + Assert.assertEquals(ErrorType.INVALID_FIELD_VALUE, exc.getErrorType()); + } + + @SuppressWarnings("null") + @Test + public void testInvalidGet2() throws OwlException { + OwlException exc = null; + try { + OwlTable tab = driver.getOwlTable(new OwlTableName(null, null)); + } catch(OwlException e) { + exc = e; + } + + Assert.assertNotNull(exc); + Assert.assertEquals(ErrorType.INVALID_FIELD_VALUE, exc.getErrorType()); + } + + @AfterClass + public static void tearDown() throws OwlException { + OwlTable tab = driver.getOwlTable(name); + + driver.dropOwlTable(tab); + List<OwlTableName> allTables = driver.showOwlTables(dbname); + Assert.assertFalse(allTables.contains(name)); + + OwlDatabase db = driver.getOwlDatabase(dbname); + driver.dropOwlDatabase(db); + } + +} Added: hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestOwlTableBuilder.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestOwlTableBuilder.java?rev=934649&view=auto ============================================================================== --- hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestOwlTableBuilder.java (added) +++ hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestOwlTableBuilder.java Thu Apr 15 23:56:44 2010 @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.owl.driver; + +import java.util.Arrays; + +import org.apache.hadoop.owl.OwlTestCase; +import org.apache.hadoop.owl.client.OwlClient; +import org.apache.hadoop.owl.common.OwlException; +import org.apache.hadoop.owl.protocol.ColumnType; +import org.apache.hadoop.owl.protocol.OwlColumnSchema; +import org.apache.hadoop.owl.protocol.OwlSchema; +import org.apache.hadoop.owl.protocol.OwlTable; +import org.apache.hadoop.owl.protocol.OwlTableName; +import org.apache.hadoop.owl.protocol.OwlKey.DataType; +import org.apache.hadoop.owl.protocol.OwlPartitionKey.IntervalFrequencyUnit; +import org.apache.hadoop.owl.testdriver.CompareOwlObjects; +import org.junit.Test; + +public class TestOwlTableBuilder extends OwlTestCase { + + private static OwlClient client; + private static OwlDriver driver; + + private static boolean initialized = false; + + public TestOwlTableBuilder() { + } + + public static void initialize() throws OwlException { + if (!initialized){ + client = new OwlClient(getUri()); + driver = new OwlDriver(getUri()); + + DriverVerificationUtil.dropOwlDatabaseIfExists(client, "testdriverdatabase"); + + client.execute("create owldatabase testdriverdatabase identified by storage directory \"testdriverdatabase\""); + initialized = true; + } + } + + public static void owlTableCreateTest(String tableName) throws Exception { + initialize(); + + DriverVerificationUtil.dropOwlTableIfExists(client, tableName, "testdriverdatabase"); + try { + + OwlSchema schema = new OwlSchema(); + schema.addColumnSchema(new OwlColumnSchema("date", ColumnType.LONG)); + schema.addColumnSchema(new OwlColumnSchema("region", ColumnType.STRING)); + schema.addColumnSchema(new OwlColumnSchema("zip", ColumnType.INT)); + + OwlTable table = new OwlTableBuilder(). + setName(new OwlTableName("testdriverdatabase", tableName)). + addProperty("size", DataType.INT). + addIntervalPartition("date", "2009-10-09 18:00:00 PDT", 15, IntervalFrequencyUnit.MINUTES). + addProperty("timezone", DataType.STRING). + addPartition("region", DataType.STRING, Arrays.asList(new String[] { "us", "uk" })). + addProperty("continent", DataType.STRING). + addPartition("zip", DataType.INT). + setSchema(schema). + build(); + + driver.createOwlTable(table); + + OwlTable driverTable = driver.getOwlTable(new OwlTableName("testdriverdatabase", tableName) ); + + CompareOwlObjects.compareObjects(table, driverTable); + + assertEquals(driverTable.getPartitionKeys().size(), 3); + assertEquals(driverTable.getPropertyKeys().size(), 3); + } finally { + client.execute("drop owltable " + tableName + " within owldatabase testdriverdatabase"); + } + } + + @Test + public static void testBoundedList() throws Exception { + initialize(); + owlTableCreateTest("testtable"); + } + +} Added: hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestSelectOwlTable.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestSelectOwlTable.java?rev=934649&view=auto ============================================================================== --- hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestSelectOwlTable.java (added) +++ hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestSelectOwlTable.java Thu Apr 15 23:56:44 2010 @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.owl.driver; + +import java.util.List; + +import org.apache.hadoop.owl.OwlTestCase; +import org.apache.hadoop.owl.common.OwlException; +import org.apache.hadoop.owl.protocol.ColumnType; +import org.apache.hadoop.owl.protocol.OwlColumnSchema; +import org.apache.hadoop.owl.protocol.OwlDatabase; +import org.apache.hadoop.owl.protocol.OwlSchema; +import org.apache.hadoop.owl.protocol.OwlTable; +import org.apache.hadoop.owl.protocol.OwlTableName; +import org.apache.hadoop.owl.protocol.OwlKey.DataType; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestSelectOwlTable { + + private static String dbname = "tsoDB"; + private static String tableName = "tsoOwltable"; + private static OwlTableName name = new OwlTableName(dbname, tableName); + + private static OwlDriver driver; + private static OwlTable table; + + public TestSelectOwlTable() { + } + + @BeforeClass + public static void setUp() throws OwlException { + driver = new OwlDriver(OwlTestCase.getUri()); + DriverVerificationUtil.dropOwlTableIfExists(driver, name); + DriverVerificationUtil.dropOwlDatabaseIfExists(driver, dbname); + String databaseLocation = "hdfs://localhost:9000/data/dummy"; + driver.createOwlDatabase(new OwlDatabase(dbname, databaseLocation)); + } + + + @Test + public void testSelectEmptyOwlTable() throws OwlException { + // select a owltable that does exist + List<OwlTableName> allTables = driver.showOwlTables(dbname, "likestring"); + Assert.assertNotNull(allTables); + Assert.assertEquals(0, allTables.size()); + } + + @AfterClass + public static void tearDown() throws OwlException { + + DriverVerificationUtil.dropOwlTableIfExists(driver, name); + DriverVerificationUtil.dropOwlDatabaseIfExists(driver, dbname); + } + +} Added: hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestSelectPartition.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestSelectPartition.java?rev=934649&view=auto ============================================================================== --- hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestSelectPartition.java (added) +++ hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/driver/TestSelectPartition.java Thu Apr 15 23:56:44 2010 @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.owl.driver; + + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.owl.OwlTestCase; +import org.apache.hadoop.owl.common.OwlException; +import org.apache.hadoop.owl.common.OwlUtil; +import org.apache.hadoop.owl.protocol.ColumnType; +import org.apache.hadoop.owl.protocol.OwlColumnSchema; +import org.apache.hadoop.owl.protocol.OwlDatabase; +import org.apache.hadoop.owl.protocol.OwlKeyValue; +import org.apache.hadoop.owl.protocol.OwlLoaderInfo; +import org.apache.hadoop.owl.protocol.OwlPartition; +import org.apache.hadoop.owl.protocol.OwlSchema; +import org.apache.hadoop.owl.protocol.OwlTable; +import org.apache.hadoop.owl.protocol.OwlTableName; +import org.apache.hadoop.owl.protocol.OwlKey.DataType; +import org.junit.Test; + +public class TestSelectPartition extends OwlTestCase{ + + private static String dbname = "testpartitiondb"; + private static String tableName = "testpartitiontab"; + + private static OwlTableName name = new OwlTableName(dbname, tableName); + private static OwlDriver driver; + + static List<OwlKeyValue> keyValues; + static OwlKeyValue k1, k2, k3; + static int count = 1; + + private static boolean initialized = false; + private static int numTestsRemaining = 6; + + public TestSelectPartition() { + } + + private static OwlLoaderInfo _instantiateOwlLoaderInfo(int i) throws OwlException{ + if ((i % 2)==0){ + return new OwlLoaderInfo("test"); + }else{ + return new OwlLoaderInfo("test",Integer.toString(i)); + } + } + + @SuppressWarnings("boxing") + static void publish(int v1, String v2, int v3) throws OwlException { + String deLocation = "hdfs://localhost/data/testdatabase/1234"; + k1.setIntValue(v1); + k2.setStringValue(v2); + k3.setIntValue(v3); + + OwlSchema schema1 = new OwlSchema(); + schema1.addColumnSchema(new OwlColumnSchema("c1", ColumnType.INT)); + schema1.addColumnSchema(new OwlColumnSchema("c2", ColumnType.STRING)); + driver.publish(name, keyValues, null, deLocation + count, schema1, _instantiateOwlLoaderInfo(v1)); + count++; + } + + public static void initialize() throws OwlException { + if (!initialized){ + driver = new OwlDriver(getUri()); + + DriverVerificationUtil.dropOwlTableIfExists(driver, name); + DriverVerificationUtil.dropOwlDatabaseIfExists(driver, dbname); + + driver.createOwlDatabase(new OwlDatabase(dbname, "hdfs://localhost:9000/data/testdatabase")); + + OwlSchema schema = new OwlSchema(); + schema.addColumnSchema(new OwlColumnSchema("date", ColumnType.LONG)); + + OwlTable table = new OwlTableBuilder(). + setName(name). + addPartition("part1", DataType.INT). + addPartition("part2", DataType.STRING). + addPartition("part3", DataType.INT). + setSchema(schema). + build(); + + driver.createOwlTable(table); + + keyValues = new ArrayList<OwlKeyValue>(); + k1 = new OwlKeyValue("part1", 1); + k2 = new OwlKeyValue("part2", "aaa"); + k3 = new OwlKeyValue("part3", 3); + keyValues.add(k1); + keyValues.add(k2); + keyValues.add(k3); + + publish(1, "aaa", 3); + publish(1, "bbb", 3); + publish(1, "bbb", 4); + publish(2, "aaa", 5); + publish(2, "bbb", 4); + + initialized = true; + } + } + + public static void runSelect(String filter, String partitionKeyName, int expectedCount) throws OwlException { + System.out.println("Filter : " + filter + " Key : " + partitionKeyName + " ExpectedCount : " + expectedCount); + + List<OwlPartition> partitions = driver.getPartitions(name, filter, partitionKeyName); + assertNotNull(partitions); + assertEquals(expectedCount, partitions.size()); + + for (OwlPartition ptn : partitions){ + OwlLoaderInfo loaderInfo = ptn.getLoader(); + if (ptn.isLeaf()){ + assertNotNull(loaderInfo); + for (OwlKeyValue kv:ptn.getKeyValues()){ + if (kv.getKeyName().equalsIgnoreCase("part1")){ + _verifyPartitionOwlLoaderInfo(loaderInfo, kv); + } + } + } + } + } + + private static void _verifyPartitionOwlLoaderInfo(OwlLoaderInfo loaderInfo, + OwlKeyValue kv) { + int i = kv.getIntValue().intValue(); + if ((i % 2) == 0){ + assertNull(loaderInfo.getInputDriverArgs()); + }else { + assertNotNull(loaderInfo.getInputDriverArgs()); + assertEquals(kv.getIntValue().toString(),loaderInfo.getInputDriverArgs()); + } + } + + @Test + public static void testSelectAll() throws OwlException { + initialize(); + runSelect(null, null, 5); + runSelect("", null, 5); + cleanup(); + } + + @Test + public static void testSelectPartitionLevel1() throws OwlException { + initialize(); + runSelect(null, "part1", 2); + runSelect("part1 = 1", "part1", 1); + cleanup(); + } + + @Test + public static void testSelectPartitionLevel2() throws OwlException { + initialize(); + runSelect(null, "part2", 4); + runSelect("part1 = 1", "part2", 2); + cleanup(); + } + + @Test + public static void testSelectPartitionLevel3() throws OwlException { + initialize(); + runSelect(null, "part3", 5); + runSelect("part1 = 1", "part3", 3); + cleanup(); + } + + @Test + public static void testSelectLeaf() throws OwlException { + initialize(); + runSelect("(part1 = 1) and (part2=\"aaa\")", null, 1); + runSelect("(part1 = 1 or part1 = 2) and part2=\"aaa\"", null, 2); + runSelect("(part1 = 1 and (part2=\"bbb\" or part3 = 4)) or (part1=2 and (part2 = \"bbb\" or part3 = 4))", null, 3); + runSelect("(part1 < 2) and (part2 >= \"aaa\")", null, 3); + runSelect("part1 = 1", null, 3); + cleanup(); + } + + @Test + public static void testSelectNotEquals() throws OwlException { + initialize(); + runSelect("part1 != 1", null, 2); + runSelect("part1 != 1", "part2", 2); + runSelect("(part1 != 1) or (part2 != \"bbb\")", "part2", 3); + runSelect("(part1 != 1) and (part2 != \"aaa\")", null, 1); + } + + @Test + public static void testDropPartition() throws OwlException { + initialize(); + driver.dropPartition(name, keyValues); //drop the last inserted partition + + //Only four partitions remain after drop + runSelect(null, null, 4); + cleanup(); + } + + public static void cleanup() throws OwlException { + numTestsRemaining--; + if (numTestsRemaining == 0){ + initialize(); + driver.dropOwlTable(new OwlTable(name)); + driver.dropOwlDatabase(new OwlDatabase(dbname, null)); + } + } +}
