http://git-wip-us.apache.org/repos/asf/hive/blob/24b366f0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java.orig ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java.orig b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java.orig new file mode 100644 index 0000000..b005759 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java.orig @@ -0,0 +1,3224 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import junit.framework.TestCase; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hive.common.FileUtils; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.api.AggrStats; +import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.ConfigValSecurityException; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Function; +import org.apache.hadoop.hive.metastore.api.FunctionType; +import org.apache.hadoop.hive.metastore.api.GetAllFunctionsResponse; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Order; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.metastore.api.ResourceType; +import org.apache.hadoop.hive.metastore.api.ResourceUri; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.SkewedInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.Type; +import org.apache.hadoop.hive.metastore.api.UnknownDBException; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.io.HiveInputFormat; +import org.apache.hadoop.hive.ql.io.HiveOutputFormat; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; +import org.apache.hadoop.util.StringUtils; +import org.apache.thrift.TException; +import org.junit.Assert; +import org.junit.Test; + +import com.google.common.collect.Lists; + +public abstract class TestHiveMetaStore extends TestCase { + private static final Log LOG = LogFactory.getLog(TestHiveMetaStore.class); + protected static HiveMetaStoreClient client; + protected static HiveConf hiveConf; + protected static Warehouse warehouse; + protected static boolean isThriftClient = false; + + private static final String TEST_DB1_NAME = "testdb1"; + private static final String TEST_DB2_NAME = "testdb2"; + + @Override + protected void setUp() throws Exception { + hiveConf = new HiveConf(this.getClass()); + warehouse = new Warehouse(hiveConf); + + // set some values to use for getting conf. vars + hiveConf.set("hive.metastore.metrics.enabled","true"); + hiveConf.set("hive.key1", "value1"); + hiveConf.set("hive.key2", "http://www.example.com"); + hiveConf.set("hive.key3", ""); + hiveConf.set("hive.key4", "0"); + + hiveConf.setIntVar(ConfVars.METASTORE_BATCH_RETRIEVE_MAX, 2); + } + + public void testNameMethods() { + Map<String, String> spec = new LinkedHashMap<String, String>(); + spec.put("ds", "2008-07-01 14:13:12"); + spec.put("hr", "14"); + List<String> vals = new ArrayList<String>(); + for(String v : spec.values()) { + vals.add(v); + } + String partName = "ds=2008-07-01 14%3A13%3A12/hr=14"; + + try { + List<String> testVals = client.partitionNameToVals(partName); + assertTrue("Values from name are incorrect", vals.equals(testVals)); + + Map<String, String> testSpec = client.partitionNameToSpec(partName); + assertTrue("Spec from name is incorrect", spec.equals(testSpec)); + + List<String> emptyVals = client.partitionNameToVals(""); + assertTrue("Values should be empty", emptyVals.size() == 0); + + Map<String, String> emptySpec = client.partitionNameToSpec(""); + assertTrue("Spec should be empty", emptySpec.size() == 0); + } catch (Exception e) { + assert(false); + } + } + + /** + * tests create table and partition and tries to drop the table without + * droppping the partition + * + * @throws Exception + */ + public void testPartition() throws Exception { + partitionTester(client, hiveConf); + } + + public static void partitionTester(HiveMetaStoreClient client, HiveConf hiveConf) + throws Exception { + try { + String dbName = "compdb"; + String tblName = "comptbl"; + String typeName = "Person"; + List<String> vals = makeVals("2008-07-01 14:13:12", "14"); + List<String> vals2 = makeVals("2008-07-01 14:13:12", "15"); + List<String> vals3 = makeVals("2008-07-02 14:13:12", "15"); + List<String> vals4 = makeVals("2008-07-03 14:13:12", "151"); + + client.dropTable(dbName, tblName); + silentDropDatabase(dbName); + Database db = new Database(); + db.setName(dbName); + client.createDatabase(db); + db = client.getDatabase(dbName); + Path dbPath = new Path(db.getLocationUri()); + FileSystem fs = FileSystem.get(dbPath.toUri(), hiveConf); + boolean inheritPerms = hiveConf.getBoolVar( + HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS); + FsPermission dbPermission = fs.getFileStatus(dbPath).getPermission(); + if (inheritPerms) { + //Set different perms for the database dir for further tests + dbPermission = new FsPermission((short)488); + fs.setPermission(dbPath, dbPermission); + } + + client.dropType(typeName); + Type typ1 = new Type(); + typ1.setName(typeName); + typ1.setFields(new ArrayList<FieldSchema>(2)); + typ1.getFields().add( + new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, "")); + typ1.getFields().add( + new FieldSchema("income", serdeConstants.INT_TYPE_NAME, "")); + client.createType(typ1); + + Table tbl = new Table(); + tbl.setDbName(dbName); + tbl.setTableName(tblName); + StorageDescriptor sd = new StorageDescriptor(); + tbl.setSd(sd); + sd.setCols(typ1.getFields()); + sd.setCompressed(false); + sd.setNumBuckets(1); + sd.setParameters(new HashMap<String, String>()); + sd.getParameters().put("test_param_1", "Use this for comments etc"); + sd.setBucketCols(new ArrayList<String>(2)); + sd.getBucketCols().add("name"); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setName(tbl.getTableName()); + sd.getSerdeInfo().setParameters(new HashMap<String, String>()); + sd.getSerdeInfo().getParameters() + .put(serdeConstants.SERIALIZATION_FORMAT, "1"); + sd.setSortCols(new ArrayList<Order>()); + sd.setStoredAsSubDirectories(false); + sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName()); + sd.setInputFormat(HiveInputFormat.class.getName()); + sd.setOutputFormat(HiveOutputFormat.class.getName()); + + //skewed information + SkewedInfo skewInfor = new SkewedInfo(); + skewInfor.setSkewedColNames(Arrays.asList("name")); + List<String> skv = Arrays.asList("1"); + skewInfor.setSkewedColValues(Arrays.asList(skv)); + Map<List<String>, String> scvlm = new HashMap<List<String>, String>(); + scvlm.put(skv, "location1"); + skewInfor.setSkewedColValueLocationMaps(scvlm); + sd.setSkewedInfo(skewInfor); + + tbl.setPartitionKeys(new ArrayList<FieldSchema>(2)); + tbl.getPartitionKeys().add( + new FieldSchema("ds", serdeConstants.STRING_TYPE_NAME, "")); + tbl.getPartitionKeys().add( + new FieldSchema("hr", serdeConstants.STRING_TYPE_NAME, "")); + + client.createTable(tbl); + + if (isThriftClient) { + // the createTable() above does not update the location in the 'tbl' + // object when the client is a thrift client and the code below relies + // on the location being present in the 'tbl' object - so get the table + // from the metastore + tbl = client.getTable(dbName, tblName); + } + + assertEquals(dbPermission, fs.getFileStatus(new Path(tbl.getSd().getLocation())) + .getPermission()); + + Partition part = makePartitionObject(dbName, tblName, vals, tbl, "/part1"); + Partition part2 = makePartitionObject(dbName, tblName, vals2, tbl, "/part2"); + Partition part3 = makePartitionObject(dbName, tblName, vals3, tbl, "/part3"); + Partition part4 = makePartitionObject(dbName, tblName, vals4, tbl, "/part4"); + + // check if the partition exists (it shouldn't) + boolean exceptionThrown = false; + try { + Partition p = client.getPartition(dbName, tblName, vals); + } catch(Exception e) { + assertEquals("partition should not have existed", + NoSuchObjectException.class, e.getClass()); + exceptionThrown = true; + } + assertTrue("getPartition() should have thrown NoSuchObjectException", exceptionThrown); + Partition retp = client.add_partition(part); + assertNotNull("Unable to create partition " + part, retp); + assertEquals(dbPermission, fs.getFileStatus(new Path(retp.getSd().getLocation())) + .getPermission()); + Partition retp2 = client.add_partition(part2); + assertNotNull("Unable to create partition " + part2, retp2); + assertEquals(dbPermission, fs.getFileStatus(new Path(retp2.getSd().getLocation())) + .getPermission()); + Partition retp3 = client.add_partition(part3); + assertNotNull("Unable to create partition " + part3, retp3); + assertEquals(dbPermission, fs.getFileStatus(new Path(retp3.getSd().getLocation())) + .getPermission()); + Partition retp4 = client.add_partition(part4); + assertNotNull("Unable to create partition " + part4, retp4); + assertEquals(dbPermission, fs.getFileStatus(new Path(retp4.getSd().getLocation())) + .getPermission()); + + Partition part_get = client.getPartition(dbName, tblName, part.getValues()); + if(isThriftClient) { + // since we are using thrift, 'part' will not have the create time and + // last DDL time set since it does not get updated in the add_partition() + // call - likewise part2 and part3 - set it correctly so that equals check + // doesn't fail + adjust(client, part, dbName, tblName); + adjust(client, part2, dbName, tblName); + adjust(client, part3, dbName, tblName); + } + assertTrue("Partitions are not same", part.equals(part_get)); + + String partName = "ds=" + FileUtils.escapePathName("2008-07-01 14:13:12") + "/hr=14"; + String part2Name = "ds=" + FileUtils.escapePathName("2008-07-01 14:13:12") + "/hr=15"; + String part3Name = "ds=" + FileUtils.escapePathName("2008-07-02 14:13:12") + "/hr=15"; + String part4Name = "ds=" + FileUtils.escapePathName("2008-07-03 14:13:12") + "/hr=151"; + + part_get = client.getPartition(dbName, tblName, partName); + assertTrue("Partitions are not the same", part.equals(part_get)); + + // Test partition listing with a partial spec - ds is specified but hr is not + List<String> partialVals = new ArrayList<String>(); + partialVals.add(vals.get(0)); + Set<Partition> parts = new HashSet<Partition>(); + parts.add(part); + parts.add(part2); + + List<Partition> partial = client.listPartitions(dbName, tblName, partialVals, + (short) -1); + assertTrue("Should have returned 2 partitions", partial.size() == 2); + assertTrue("Not all parts returned", partial.containsAll(parts)); + + Set<String> partNames = new HashSet<String>(); + partNames.add(partName); + partNames.add(part2Name); + List<String> partialNames = client.listPartitionNames(dbName, tblName, partialVals, + (short) -1); + assertTrue("Should have returned 2 partition names", partialNames.size() == 2); + assertTrue("Not all part names returned", partialNames.containsAll(partNames)); + + partNames.add(part3Name); + partNames.add(part4Name); + partialVals.clear(); + partialVals.add(""); + partialNames = client.listPartitionNames(dbName, tblName, partialVals, (short) -1); + assertTrue("Should have returned 4 partition names", partialNames.size() == 4); + assertTrue("Not all part names returned", partialNames.containsAll(partNames)); + + // Test partition listing with a partial spec - hr is specified but ds is not + parts.clear(); + parts.add(part2); + parts.add(part3); + + partialVals.clear(); + partialVals.add(""); + partialVals.add(vals2.get(1)); + + partial = client.listPartitions(dbName, tblName, partialVals, (short) -1); + assertEquals("Should have returned 2 partitions", 2, partial.size()); + assertTrue("Not all parts returned", partial.containsAll(parts)); + + partNames.clear(); + partNames.add(part2Name); + partNames.add(part3Name); + partialNames = client.listPartitionNames(dbName, tblName, partialVals, + (short) -1); + assertEquals("Should have returned 2 partition names", 2, partialNames.size()); + assertTrue("Not all part names returned", partialNames.containsAll(partNames)); + + // Verify escaped partition names don't return partitions + exceptionThrown = false; + try { + String badPartName = "ds=2008-07-01 14%3A13%3A12/hrs=14"; + client.getPartition(dbName, tblName, badPartName); + } catch(NoSuchObjectException e) { + exceptionThrown = true; + } + assertTrue("Bad partition spec should have thrown an exception", exceptionThrown); + + Path partPath = new Path(part.getSd().getLocation()); + + + assertTrue(fs.exists(partPath)); + client.dropPartition(dbName, tblName, part.getValues(), true); + assertFalse(fs.exists(partPath)); + + // Test append_partition_by_name + client.appendPartition(dbName, tblName, partName); + Partition part5 = client.getPartition(dbName, tblName, part.getValues()); + assertTrue("Append partition by name failed", part5.getValues().equals(vals));; + Path part5Path = new Path(part5.getSd().getLocation()); + assertTrue(fs.exists(part5Path)); + + // Test drop_partition_by_name + assertTrue("Drop partition by name failed", + client.dropPartition(dbName, tblName, partName, true)); + assertFalse(fs.exists(part5Path)); + + // add the partition again so that drop table with a partition can be + // tested + retp = client.add_partition(part); + assertNotNull("Unable to create partition " + part, retp); + assertEquals(dbPermission, fs.getFileStatus(new Path(retp.getSd().getLocation())) + .getPermission()); + + // test add_partitions + + List<String> mvals1 = makeVals("2008-07-04 14:13:12", "14641"); + List<String> mvals2 = makeVals("2008-07-04 14:13:12", "14642"); + List<String> mvals3 = makeVals("2008-07-04 14:13:12", "14643"); + List<String> mvals4 = makeVals("2008-07-04 14:13:12", "14643"); // equal to 3 + List<String> mvals5 = makeVals("2008-07-04 14:13:12", "14645"); + + Exception savedException; + + // add_partitions(empty list) : ok, normal operation + client.add_partitions(new ArrayList<Partition>()); + + // add_partitions(1,2,3) : ok, normal operation + Partition mpart1 = makePartitionObject(dbName, tblName, mvals1, tbl, "/mpart1"); + Partition mpart2 = makePartitionObject(dbName, tblName, mvals2, tbl, "/mpart2"); + Partition mpart3 = makePartitionObject(dbName, tblName, mvals3, tbl, "/mpart3"); + client.add_partitions(Arrays.asList(mpart1,mpart2,mpart3)); + + if(isThriftClient) { + // do DDL time munging if thrift mode + adjust(client, mpart1, dbName, tblName); + adjust(client, mpart2, dbName, tblName); + adjust(client, mpart3, dbName, tblName); + } + verifyPartitionsPublished(client, dbName, tblName, + Arrays.asList(mvals1.get(0)), + Arrays.asList(mpart1,mpart2,mpart3)); + + Partition mpart4 = makePartitionObject(dbName, tblName, mvals4, tbl, "/mpart4"); + Partition mpart5 = makePartitionObject(dbName, tblName, mvals5, tbl, "/mpart5"); + + // create dir for /mpart5 + Path mp5Path = new Path(mpart5.getSd().getLocation()); + warehouse.mkdirs(mp5Path, true); + assertTrue(fs.exists(mp5Path)); + assertEquals(dbPermission, fs.getFileStatus(mp5Path).getPermission()); + + // add_partitions(5,4) : err = duplicate keyvals on mpart4 + savedException = null; + try { + client.add_partitions(Arrays.asList(mpart5,mpart4)); + } catch (Exception e) { + savedException = e; + } finally { + assertNotNull(savedException); + } + + // check that /mpart4 does not exist, but /mpart5 still does. + assertTrue(fs.exists(mp5Path)); + assertFalse(fs.exists(new Path(mpart4.getSd().getLocation()))); + + // add_partitions(5) : ok + client.add_partitions(Arrays.asList(mpart5)); + + if(isThriftClient) { + // do DDL time munging if thrift mode + adjust(client, mpart5, dbName, tblName); + } + + verifyPartitionsPublished(client, dbName, tblName, + Arrays.asList(mvals1.get(0)), + Arrays.asList(mpart1,mpart2,mpart3,mpart5)); + + //// end add_partitions tests + + client.dropTable(dbName, tblName); + + client.dropType(typeName); + + // recreate table as external, drop partition and it should + // still exist + tbl.setParameters(new HashMap<String, String>()); + tbl.getParameters().put("EXTERNAL", "TRUE"); + client.createTable(tbl); + retp = client.add_partition(part); + assertTrue(fs.exists(partPath)); + client.dropPartition(dbName, tblName, part.getValues(), true); + assertTrue(fs.exists(partPath)); + + for (String tableName : client.getTables(dbName, "*")) { + client.dropTable(dbName, tableName); + } + + client.dropDatabase(dbName); + + } catch (Exception e) { + System.err.println(StringUtils.stringifyException(e)); + System.err.println("testPartition() failed."); + throw e; + } + } + + private static void verifyPartitionsPublished(HiveMetaStoreClient client, + String dbName, String tblName, List<String> partialSpec, + List<Partition> expectedPartitions) + throws NoSuchObjectException, MetaException, TException { + // Test partition listing with a partial spec + + List<Partition> mpartial = client.listPartitions(dbName, tblName, partialSpec, + (short) -1); + assertEquals("Should have returned "+expectedPartitions.size()+ + " partitions, returned " + mpartial.size(), + expectedPartitions.size(), mpartial.size()); + assertTrue("Not all parts returned", mpartial.containsAll(expectedPartitions)); + } + + private static List<String> makeVals(String ds, String id) { + List <String> vals4 = new ArrayList<String>(2); + vals4 = new ArrayList<String>(2); + vals4.add(ds); + vals4.add(id); + return vals4; + } + + private static Partition makePartitionObject(String dbName, String tblName, + List<String> ptnVals, Table tbl, String ptnLocationSuffix) throws MetaException { + Partition part4 = new Partition(); + part4.setDbName(dbName); + part4.setTableName(tblName); + part4.setValues(ptnVals); + part4.setParameters(new HashMap<String, String>()); + part4.setSd(tbl.getSd().deepCopy()); + part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy()); + part4.getSd().setLocation(tbl.getSd().getLocation() + ptnLocationSuffix); + MetaStoreUtils.updatePartitionStatsFast(part4, warehouse); + return part4; + } + + public void testListPartitions() throws Throwable { + // create a table with multiple partitions + String dbName = "compdb"; + String tblName = "comptbl"; + String typeName = "Person"; + + cleanUp(dbName, tblName, typeName); + + List<List<String>> values = new ArrayList<List<String>>(); + values.add(makeVals("2008-07-01 14:13:12", "14")); + values.add(makeVals("2008-07-01 14:13:12", "15")); + values.add(makeVals("2008-07-02 14:13:12", "15")); + values.add(makeVals("2008-07-03 14:13:12", "151")); + + createMultiPartitionTableSchema(dbName, tblName, typeName, values); + + List<Partition> partitions = client.listPartitions(dbName, tblName, (short)-1); + assertNotNull("should have returned partitions", partitions); + assertEquals(" should have returned " + values.size() + + " partitions", values.size(), partitions.size()); + + partitions = client.listPartitions(dbName, tblName, (short)(values.size()/2)); + + assertNotNull("should have returned partitions", partitions); + assertEquals(" should have returned " + values.size() / 2 + + " partitions",values.size() / 2, partitions.size()); + + + partitions = client.listPartitions(dbName, tblName, (short) (values.size() * 2)); + + assertNotNull("should have returned partitions", partitions); + assertEquals(" should have returned " + values.size() + + " partitions",values.size(), partitions.size()); + + cleanUp(dbName, tblName, typeName); + + } + + + + public void testListPartitionNames() throws Throwable { + // create a table with multiple partitions + String dbName = "compdb"; + String tblName = "comptbl"; + String typeName = "Person"; + + cleanUp(dbName, tblName, typeName); + + List<List<String>> values = new ArrayList<List<String>>(); + values.add(makeVals("2008-07-01 14:13:12", "14")); + values.add(makeVals("2008-07-01 14:13:12", "15")); + values.add(makeVals("2008-07-02 14:13:12", "15")); + values.add(makeVals("2008-07-03 14:13:12", "151")); + + + + createMultiPartitionTableSchema(dbName, tblName, typeName, values); + + List<String> partitions = client.listPartitionNames(dbName, tblName, (short)-1); + assertNotNull("should have returned partitions", partitions); + assertEquals(" should have returned " + values.size() + + " partitions", values.size(), partitions.size()); + + partitions = client.listPartitionNames(dbName, tblName, (short)(values.size()/2)); + + assertNotNull("should have returned partitions", partitions); + assertEquals(" should have returned " + values.size() / 2 + + " partitions",values.size() / 2, partitions.size()); + + + partitions = client.listPartitionNames(dbName, tblName, (short) (values.size() * 2)); + + assertNotNull("should have returned partitions", partitions); + assertEquals(" should have returned " + values.size() + + " partitions",values.size(), partitions.size()); + + cleanUp(dbName, tblName, typeName); + + } + + + public void testDropTable() throws Throwable { + // create a table with multiple partitions + String dbName = "compdb"; + String tblName = "comptbl"; + String typeName = "Person"; + + cleanUp(dbName, tblName, typeName); + + List<List<String>> values = new ArrayList<List<String>>(); + values.add(makeVals("2008-07-01 14:13:12", "14")); + values.add(makeVals("2008-07-01 14:13:12", "15")); + values.add(makeVals("2008-07-02 14:13:12", "15")); + values.add(makeVals("2008-07-03 14:13:12", "151")); + + createMultiPartitionTableSchema(dbName, tblName, typeName, values); + + client.dropTable(dbName, tblName); + client.dropType(typeName); + + boolean exceptionThrown = false; + try { + client.getTable(dbName, tblName); + } catch(Exception e) { + assertEquals("table should not have existed", + NoSuchObjectException.class, e.getClass()); + exceptionThrown = true; + } + assertTrue("Table " + tblName + " should have been dropped ", exceptionThrown); + + } + + public void testAlterViewParititon() throws Throwable { + String dbName = "compdb"; + String tblName = "comptbl"; + String viewName = "compView"; + + client.dropTable(dbName, tblName); + silentDropDatabase(dbName); + Database db = new Database(); + db.setName(dbName); + db.setDescription("Alter Partition Test database"); + client.createDatabase(db); + + ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(2); + cols.add(new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, "")); + cols.add(new FieldSchema("income", serdeConstants.INT_TYPE_NAME, "")); + + Table tbl = new Table(); + tbl.setDbName(dbName); + tbl.setTableName(tblName); + StorageDescriptor sd = new StorageDescriptor(); + tbl.setSd(sd); + sd.setCols(cols); + sd.setCompressed(false); + sd.setParameters(new HashMap<String, String>()); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setName(tbl.getTableName()); + sd.getSerdeInfo().setParameters(new HashMap<String, String>()); + sd.getSerdeInfo().getParameters() + .put(serdeConstants.SERIALIZATION_FORMAT, "1"); + sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName()); + sd.setInputFormat(HiveInputFormat.class.getName()); + sd.setOutputFormat(HiveOutputFormat.class.getName()); + sd.setSortCols(new ArrayList<Order>()); + + client.createTable(tbl); + + if (isThriftClient) { + // the createTable() above does not update the location in the 'tbl' + // object when the client is a thrift client and the code below relies + // on the location being present in the 'tbl' object - so get the table + // from the metastore + tbl = client.getTable(dbName, tblName); + } + + ArrayList<FieldSchema> viewCols = new ArrayList<FieldSchema>(1); + viewCols.add(new FieldSchema("income", serdeConstants.INT_TYPE_NAME, "")); + + ArrayList<FieldSchema> viewPartitionCols = new ArrayList<FieldSchema>(1); + viewPartitionCols.add(new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, "")); + + Table view = new Table(); + view.setDbName(dbName); + view.setTableName(viewName); + view.setTableType(TableType.VIRTUAL_VIEW.name()); + view.setPartitionKeys(viewPartitionCols); + view.setViewOriginalText("SELECT income, name FROM " + tblName); + view.setViewExpandedText("SELECT `" + tblName + "`.`income`, `" + tblName + + "`.`name` FROM `" + dbName + "`.`" + tblName + "`"); + StorageDescriptor viewSd = new StorageDescriptor(); + view.setSd(viewSd); + viewSd.setCols(viewCols); + viewSd.setCompressed(false); + viewSd.setParameters(new HashMap<String, String>()); + viewSd.setSerdeInfo(new SerDeInfo()); + viewSd.getSerdeInfo().setParameters(new HashMap<String, String>()); + + client.createTable(view); + + if (isThriftClient) { + // the createTable() above does not update the location in the 'tbl' + // object when the client is a thrift client and the code below relies + // on the location being present in the 'tbl' object - so get the table + // from the metastore + view = client.getTable(dbName, viewName); + } + + List<String> vals = new ArrayList<String>(1); + vals.add("abc"); + + Partition part = new Partition(); + part.setDbName(dbName); + part.setTableName(viewName); + part.setValues(vals); + part.setParameters(new HashMap<String, String>()); + + client.add_partition(part); + + Partition part2 = client.getPartition(dbName, viewName, part.getValues()); + + part2.getParameters().put("a", "b"); + + client.alter_partition(dbName, viewName, part2); + + Partition part3 = client.getPartition(dbName, viewName, part.getValues()); + assertEquals("couldn't view alter partition", part3.getParameters().get( + "a"), "b"); + + client.dropTable(dbName, viewName); + + client.dropTable(dbName, tblName); + + client.dropDatabase(dbName); + } + + public void testAlterPartition() throws Throwable { + + try { + String dbName = "compdb"; + String tblName = "comptbl"; + List<String> vals = new ArrayList<String>(2); + vals.add("2008-07-01"); + vals.add("14"); + + client.dropTable(dbName, tblName); + silentDropDatabase(dbName); + Database db = new Database(); + db.setName(dbName); + db.setDescription("Alter Partition Test database"); + client.createDatabase(db); + + ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(2); + cols.add(new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, "")); + cols.add(new FieldSchema("income", serdeConstants.INT_TYPE_NAME, "")); + + Table tbl = new Table(); + tbl.setDbName(dbName); + tbl.setTableName(tblName); + StorageDescriptor sd = new StorageDescriptor(); + tbl.setSd(sd); + sd.setCols(cols); + sd.setCompressed(false); + sd.setNumBuckets(1); + sd.setParameters(new HashMap<String, String>()); + sd.getParameters().put("test_param_1", "Use this for comments etc"); + sd.setBucketCols(new ArrayList<String>(2)); + sd.getBucketCols().add("name"); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setName(tbl.getTableName()); + sd.getSerdeInfo().setParameters(new HashMap<String, String>()); + sd.getSerdeInfo().getParameters() + .put(serdeConstants.SERIALIZATION_FORMAT, "1"); + sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName()); + sd.setInputFormat(HiveInputFormat.class.getName()); + sd.setOutputFormat(HiveOutputFormat.class.getName()); + sd.setSortCols(new ArrayList<Order>()); + + tbl.setPartitionKeys(new ArrayList<FieldSchema>(2)); + tbl.getPartitionKeys().add( + new FieldSchema("ds", serdeConstants.STRING_TYPE_NAME, "")); + tbl.getPartitionKeys().add( + new FieldSchema("hr", serdeConstants.INT_TYPE_NAME, "")); + + client.createTable(tbl); + + if (isThriftClient) { + // the createTable() above does not update the location in the 'tbl' + // object when the client is a thrift client and the code below relies + // on the location being present in the 'tbl' object - so get the table + // from the metastore + tbl = client.getTable(dbName, tblName); + } + + Partition part = new Partition(); + part.setDbName(dbName); + part.setTableName(tblName); + part.setValues(vals); + part.setParameters(new HashMap<String, String>()); + part.setSd(tbl.getSd()); + part.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo()); + part.getSd().setLocation(tbl.getSd().getLocation() + "/part1"); + + client.add_partition(part); + + Partition part2 = client.getPartition(dbName, tblName, part.getValues()); + + part2.getParameters().put("retention", "10"); + part2.getSd().setNumBuckets(12); + part2.getSd().getSerdeInfo().getParameters().put("abc", "1"); + client.alter_partition(dbName, tblName, part2); + + Partition part3 = client.getPartition(dbName, tblName, part.getValues()); + assertEquals("couldn't alter partition", part3.getParameters().get( + "retention"), "10"); + assertEquals("couldn't alter partition", part3.getSd().getSerdeInfo() + .getParameters().get("abc"), "1"); + assertEquals("couldn't alter partition", part3.getSd().getNumBuckets(), + 12); + + client.dropTable(dbName, tblName); + + client.dropDatabase(dbName); + } catch (Exception e) { + System.err.println(StringUtils.stringifyException(e)); + System.err.println("testPartition() failed."); + throw e; + } + } + + public void testRenamePartition() throws Throwable { + + try { + String dbName = "compdb1"; + String tblName = "comptbl1"; + List<String> vals = new ArrayList<String>(2); + vals.add("2011-07-11"); + vals.add("8"); + String part_path = "/ds=2011-07-11/hr=8"; + List<String> tmp_vals = new ArrayList<String>(2); + tmp_vals.add("tmp_2011-07-11"); + tmp_vals.add("-8"); + String part2_path = "/ds=tmp_2011-07-11/hr=-8"; + + client.dropTable(dbName, tblName); + silentDropDatabase(dbName); + Database db = new Database(); + db.setName(dbName); + db.setDescription("Rename Partition Test database"); + client.createDatabase(db); + + ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(2); + cols.add(new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, "")); + cols.add(new FieldSchema("income", serdeConstants.INT_TYPE_NAME, "")); + + Table tbl = new Table(); + tbl.setDbName(dbName); + tbl.setTableName(tblName); + StorageDescriptor sd = new StorageDescriptor(); + tbl.setSd(sd); + sd.setCols(cols); + sd.setCompressed(false); + sd.setNumBuckets(1); + sd.setParameters(new HashMap<String, String>()); + sd.getParameters().put("test_param_1", "Use this for comments etc"); + sd.setBucketCols(new ArrayList<String>(2)); + sd.getBucketCols().add("name"); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setName(tbl.getTableName()); + sd.getSerdeInfo().setParameters(new HashMap<String, String>()); + sd.getSerdeInfo().getParameters() + .put(serdeConstants.SERIALIZATION_FORMAT, "1"); + sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName()); + sd.setInputFormat(HiveInputFormat.class.getName()); + sd.setOutputFormat(HiveOutputFormat.class.getName()); + sd.setSortCols(new ArrayList<Order>()); + + tbl.setPartitionKeys(new ArrayList<FieldSchema>(2)); + tbl.getPartitionKeys().add( + new FieldSchema("ds", serdeConstants.STRING_TYPE_NAME, "")); + tbl.getPartitionKeys().add( + new FieldSchema("hr", serdeConstants.INT_TYPE_NAME, "")); + + client.createTable(tbl); + + if (isThriftClient) { + // the createTable() above does not update the location in the 'tbl' + // object when the client is a thrift client and the code below relies + // on the location being present in the 'tbl' object - so get the table + // from the metastore + tbl = client.getTable(dbName, tblName); + } + + Partition part = new Partition(); + part.setDbName(dbName); + part.setTableName(tblName); + part.setValues(vals); + part.setParameters(new HashMap<String, String>()); + part.setSd(tbl.getSd().deepCopy()); + part.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo()); + part.getSd().setLocation(tbl.getSd().getLocation() + "/part1"); + part.getParameters().put("retention", "10"); + part.getSd().setNumBuckets(12); + part.getSd().getSerdeInfo().getParameters().put("abc", "1"); + + client.add_partition(part); + + part.setValues(tmp_vals); + client.renamePartition(dbName, tblName, vals, part); + + boolean exceptionThrown = false; + try { + Partition p = client.getPartition(dbName, tblName, vals); + } catch(Exception e) { + assertEquals("partition should not have existed", + NoSuchObjectException.class, e.getClass()); + exceptionThrown = true; + } + assertTrue("Expected NoSuchObjectException", exceptionThrown); + + Partition part3 = client.getPartition(dbName, tblName, tmp_vals); + assertEquals("couldn't rename partition", part3.getParameters().get( + "retention"), "10"); + assertEquals("couldn't rename partition", part3.getSd().getSerdeInfo() + .getParameters().get("abc"), "1"); + assertEquals("couldn't rename partition", part3.getSd().getNumBuckets(), + 12); + assertEquals("new partition sd matches", part3.getSd().getLocation(), + tbl.getSd().getLocation() + part2_path); + + part.setValues(vals); + client.renamePartition(dbName, tblName, tmp_vals, part); + + exceptionThrown = false; + try { + Partition p = client.getPartition(dbName, tblName, tmp_vals); + } catch(Exception e) { + assertEquals("partition should not have existed", + NoSuchObjectException.class, e.getClass()); + exceptionThrown = true; + } + assertTrue("Expected NoSuchObjectException", exceptionThrown); + + part3 = client.getPartition(dbName, tblName, vals); + assertEquals("couldn't rename partition", part3.getParameters().get( + "retention"), "10"); + assertEquals("couldn't rename partition", part3.getSd().getSerdeInfo() + .getParameters().get("abc"), "1"); + assertEquals("couldn't rename partition", part3.getSd().getNumBuckets(), + 12); + assertEquals("new partition sd matches", part3.getSd().getLocation(), + tbl.getSd().getLocation() + part_path); + + client.dropTable(dbName, tblName); + + client.dropDatabase(dbName); + } catch (Exception e) { + System.err.println(StringUtils.stringifyException(e)); + System.err.println("testRenamePartition() failed."); + throw e; + } + } + + public void testDatabase() throws Throwable { + try { + // clear up any existing databases + silentDropDatabase(TEST_DB1_NAME); + silentDropDatabase(TEST_DB2_NAME); + + Database db = new Database(); + db.setName(TEST_DB1_NAME); + db.setOwnerName(SessionState.getUserFromAuthenticator()); + db.setOwnerType(PrincipalType.USER); + client.createDatabase(db); + + db = client.getDatabase(TEST_DB1_NAME); + + assertEquals("name of returned db is different from that of inserted db", + TEST_DB1_NAME, db.getName()); + assertEquals("location of the returned db is different from that of inserted db", + warehouse.getDatabasePath(db).toString(), db.getLocationUri()); + assertEquals(db.getOwnerName(), SessionState.getUserFromAuthenticator()); + assertEquals(db.getOwnerType(), PrincipalType.USER); + Database db2 = new Database(); + db2.setName(TEST_DB2_NAME); + client.createDatabase(db2); + + db2 = client.getDatabase(TEST_DB2_NAME); + + assertEquals("name of returned db is different from that of inserted db", + TEST_DB2_NAME, db2.getName()); + assertEquals("location of the returned db is different from that of inserted db", + warehouse.getDatabasePath(db2).toString(), db2.getLocationUri()); + + List<String> dbs = client.getDatabases(".*"); + + assertTrue("first database is not " + TEST_DB1_NAME, dbs.contains(TEST_DB1_NAME)); + assertTrue("second database is not " + TEST_DB2_NAME, dbs.contains(TEST_DB2_NAME)); + + client.dropDatabase(TEST_DB1_NAME); + client.dropDatabase(TEST_DB2_NAME); + silentDropDatabase(TEST_DB1_NAME); + silentDropDatabase(TEST_DB2_NAME); + } catch (Throwable e) { + System.err.println(StringUtils.stringifyException(e)); + System.err.println("testDatabase() failed."); + throw e; + } + } + + public void testDatabaseLocationWithPermissionProblems() throws Exception { + + // Note: The following test will fail if you are running this test as root. Setting + // permission to '0' on the database folder will not preclude root from being able + // to create the necessary files. + + if (System.getProperty("user.name").equals("root")) { + System.err.println("Skipping test because you are running as root!"); + return; + } + + silentDropDatabase(TEST_DB1_NAME); + + Database db = new Database(); + db.setName(TEST_DB1_NAME); + String dbLocation = + HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTOREWAREHOUSE) + "/test/_testDB_create_"; + FileSystem fs = FileSystem.get(new Path(dbLocation).toUri(), hiveConf); + fs.mkdirs( + new Path(HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTOREWAREHOUSE) + "/test"), + new FsPermission((short) 0)); + db.setLocationUri(dbLocation); + + + boolean createFailed = false; + try { + client.createDatabase(db); + } catch (MetaException cantCreateDB) { + createFailed = true; + } finally { + // Cleanup + if (!createFailed) { + try { + client.dropDatabase(TEST_DB1_NAME); + } catch(Exception e) { + System.err.println("Failed to remove database in cleanup: " + e.getMessage()); + } + } + + fs.setPermission(new Path(HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTOREWAREHOUSE) + "/test"), + new FsPermission((short) 755)); + fs.delete(new Path(HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTOREWAREHOUSE) + "/test"), true); + } + + assertTrue("Database creation succeeded even with permission problem", createFailed); + } + + public void testDatabaseLocation() throws Throwable { + try { + // clear up any existing databases + silentDropDatabase(TEST_DB1_NAME); + + Database db = new Database(); + db.setName(TEST_DB1_NAME); + String dbLocation = + HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTOREWAREHOUSE) + "/_testDB_create_"; + db.setLocationUri(dbLocation); + client.createDatabase(db); + + db = client.getDatabase(TEST_DB1_NAME); + + assertEquals("name of returned db is different from that of inserted db", + TEST_DB1_NAME, db.getName()); + assertEquals("location of the returned db is different from that of inserted db", + warehouse.getDnsPath(new Path(dbLocation)).toString(), db.getLocationUri()); + + client.dropDatabase(TEST_DB1_NAME); + silentDropDatabase(TEST_DB1_NAME); + + boolean objectNotExist = false; + try { + client.getDatabase(TEST_DB1_NAME); + } catch (NoSuchObjectException e) { + objectNotExist = true; + } + assertTrue("Database " + TEST_DB1_NAME + " exists ", objectNotExist); + + db = new Database(); + db.setName(TEST_DB1_NAME); + dbLocation = + HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTOREWAREHOUSE) + "/_testDB_file_"; + FileSystem fs = FileSystem.get(new Path(dbLocation).toUri(), hiveConf); + fs.createNewFile(new Path(dbLocation)); + fs.deleteOnExit(new Path(dbLocation)); + db.setLocationUri(dbLocation); + + boolean createFailed = false; + try { + client.createDatabase(db); + } catch (MetaException cantCreateDB) { + System.err.println(cantCreateDB.getMessage()); + createFailed = true; + } + assertTrue("Database creation succeeded even location exists and is a file", createFailed); + + objectNotExist = false; + try { + client.getDatabase(TEST_DB1_NAME); + } catch (NoSuchObjectException e) { + objectNotExist = true; + } + assertTrue("Database " + TEST_DB1_NAME + " exists when location is specified and is a file", + objectNotExist); + + } catch (Throwable e) { + System.err.println(StringUtils.stringifyException(e)); + System.err.println("testDatabaseLocation() failed."); + throw e; + } + } + + + public void testSimpleTypeApi() throws Exception { + try { + client.dropType(serdeConstants.INT_TYPE_NAME); + + Type typ1 = new Type(); + typ1.setName(serdeConstants.INT_TYPE_NAME); + boolean ret = client.createType(typ1); + assertTrue("Unable to create type", ret); + + Type typ1_2 = client.getType(serdeConstants.INT_TYPE_NAME); + assertNotNull(typ1_2); + assertEquals(typ1.getName(), typ1_2.getName()); + + ret = client.dropType(serdeConstants.INT_TYPE_NAME); + assertTrue("unable to drop type integer", ret); + + boolean exceptionThrown = false; + try { + client.getType(serdeConstants.INT_TYPE_NAME); + } catch (NoSuchObjectException e) { + exceptionThrown = true; + } + assertTrue("Expected NoSuchObjectException", exceptionThrown); + } catch (Exception e) { + System.err.println(StringUtils.stringifyException(e)); + System.err.println("testSimpleTypeApi() failed."); + throw e; + } + } + + // TODO:pc need to enhance this with complex fields and getType_all function + public void testComplexTypeApi() throws Exception { + try { + client.dropType("Person"); + + Type typ1 = new Type(); + typ1.setName("Person"); + typ1.setFields(new ArrayList<FieldSchema>(2)); + typ1.getFields().add( + new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, "")); + typ1.getFields().add( + new FieldSchema("income", serdeConstants.INT_TYPE_NAME, "")); + boolean ret = client.createType(typ1); + assertTrue("Unable to create type", ret); + + Type typ1_2 = client.getType("Person"); + assertNotNull("type Person not found", typ1_2); + assertEquals(typ1.getName(), typ1_2.getName()); + assertEquals(typ1.getFields().size(), typ1_2.getFields().size()); + assertEquals(typ1.getFields().get(0), typ1_2.getFields().get(0)); + assertEquals(typ1.getFields().get(1), typ1_2.getFields().get(1)); + + client.dropType("Family"); + + Type fam = new Type(); + fam.setName("Family"); + fam.setFields(new ArrayList<FieldSchema>(2)); + fam.getFields().add( + new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, "")); + fam.getFields().add( + new FieldSchema("members", + MetaStoreUtils.getListType(typ1.getName()), "")); + + ret = client.createType(fam); + assertTrue("Unable to create type " + fam.getName(), ret); + + Type fam2 = client.getType("Family"); + assertNotNull("type Person not found", fam2); + assertEquals(fam.getName(), fam2.getName()); + assertEquals(fam.getFields().size(), fam2.getFields().size()); + assertEquals(fam.getFields().get(0), fam2.getFields().get(0)); + assertEquals(fam.getFields().get(1), fam2.getFields().get(1)); + + ret = client.dropType("Family"); + assertTrue("unable to drop type Family", ret); + + ret = client.dropType("Person"); + assertTrue("unable to drop type Person", ret); + + boolean exceptionThrown = false; + try { + client.getType("Person"); + } catch (NoSuchObjectException e) { + exceptionThrown = true; + } + assertTrue("Expected NoSuchObjectException", exceptionThrown); + } catch (Exception e) { + System.err.println(StringUtils.stringifyException(e)); + System.err.println("testComplexTypeApi() failed."); + throw e; + } + } + + public void testSimpleTable() throws Exception { + try { + String dbName = "simpdb"; + String tblName = "simptbl"; + String tblName2 = "simptbl2"; + String typeName = "Person"; + + client.dropTable(dbName, tblName); + silentDropDatabase(dbName); + + Database db = new Database(); + db.setName(dbName); + client.createDatabase(db); + + client.dropType(typeName); + Type typ1 = new Type(); + typ1.setName(typeName); + typ1.setFields(new ArrayList<FieldSchema>(2)); + typ1.getFields().add( + new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, "")); + typ1.getFields().add( + new FieldSchema("income", serdeConstants.INT_TYPE_NAME, "")); + client.createType(typ1); + + Table tbl = new Table(); + tbl.setDbName(dbName); + tbl.setTableName(tblName); + StorageDescriptor sd = new StorageDescriptor(); + tbl.setSd(sd); + sd.setCols(typ1.getFields()); + sd.setCompressed(false); + sd.setNumBuckets(1); + sd.setParameters(new HashMap<String, String>()); + sd.getParameters().put("test_param_1", "Use this for comments etc"); + sd.setBucketCols(new ArrayList<String>(2)); + sd.getBucketCols().add("name"); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setName(tbl.getTableName()); + sd.getSerdeInfo().setParameters(new HashMap<String, String>()); + sd.getSerdeInfo().getParameters().put( + org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1"); + sd.getSerdeInfo().setSerializationLib( + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); + sd.setInputFormat(HiveInputFormat.class.getName()); + sd.setInputFormat(HiveOutputFormat.class.getName()); + + tbl.setPartitionKeys(new ArrayList<FieldSchema>()); + + client.createTable(tbl); + + if (isThriftClient) { + // the createTable() above does not update the location in the 'tbl' + // object when the client is a thrift client and the code below relies + // on the location being present in the 'tbl' object - so get the table + // from the metastore + tbl = client.getTable(dbName, tblName); + } + + Table tbl2 = client.getTable(dbName, tblName); + assertNotNull(tbl2); + assertEquals(tbl2.getDbName(), dbName); + assertEquals(tbl2.getTableName(), tblName); + assertEquals(tbl2.getSd().getCols().size(), typ1.getFields().size()); + assertEquals(tbl2.getSd().isCompressed(), false); + assertEquals(tbl2.getSd().getNumBuckets(), 1); + assertEquals(tbl2.getSd().getLocation(), tbl.getSd().getLocation()); + assertNotNull(tbl2.getSd().getSerdeInfo()); + sd.getSerdeInfo().setParameters(new HashMap<String, String>()); + sd.getSerdeInfo().getParameters().put( + org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1"); + + tbl2.setTableName(tblName2); + tbl2.setParameters(new HashMap<String, String>()); + tbl2.getParameters().put("EXTERNAL", "TRUE"); + tbl2.getSd().setLocation(tbl.getSd().getLocation() + "-2"); + + List<FieldSchema> fieldSchemas = client.getFields(dbName, tblName); + assertNotNull(fieldSchemas); + assertEquals(fieldSchemas.size(), tbl.getSd().getCols().size()); + for (FieldSchema fs : tbl.getSd().getCols()) { + assertTrue(fieldSchemas.contains(fs)); + } + + List<FieldSchema> fieldSchemasFull = client.getSchema(dbName, tblName); + assertNotNull(fieldSchemasFull); + assertEquals(fieldSchemasFull.size(), tbl.getSd().getCols().size() + + tbl.getPartitionKeys().size()); + for (FieldSchema fs : tbl.getSd().getCols()) { + assertTrue(fieldSchemasFull.contains(fs)); + } + for (FieldSchema fs : tbl.getPartitionKeys()) { + assertTrue(fieldSchemasFull.contains(fs)); + } + + client.createTable(tbl2); + if (isThriftClient) { + tbl2 = client.getTable(tbl2.getDbName(), tbl2.getTableName()); + } + + Table tbl3 = client.getTable(dbName, tblName2); + assertNotNull(tbl3); + assertEquals(tbl3.getDbName(), dbName); + assertEquals(tbl3.getTableName(), tblName2); + assertEquals(tbl3.getSd().getCols().size(), typ1.getFields().size()); + assertEquals(tbl3.getSd().isCompressed(), false); + assertEquals(tbl3.getSd().getNumBuckets(), 1); + assertEquals(tbl3.getSd().getLocation(), tbl2.getSd().getLocation()); + assertEquals(tbl3.getParameters(), tbl2.getParameters()); + + fieldSchemas = client.getFields(dbName, tblName2); + assertNotNull(fieldSchemas); + assertEquals(fieldSchemas.size(), tbl2.getSd().getCols().size()); + for (FieldSchema fs : tbl2.getSd().getCols()) { + assertTrue(fieldSchemas.contains(fs)); + } + + fieldSchemasFull = client.getSchema(dbName, tblName2); + assertNotNull(fieldSchemasFull); + assertEquals(fieldSchemasFull.size(), tbl2.getSd().getCols().size() + + tbl2.getPartitionKeys().size()); + for (FieldSchema fs : tbl2.getSd().getCols()) { + assertTrue(fieldSchemasFull.contains(fs)); + } + for (FieldSchema fs : tbl2.getPartitionKeys()) { + assertTrue(fieldSchemasFull.contains(fs)); + } + + assertEquals("Use this for comments etc", tbl2.getSd().getParameters() + .get("test_param_1")); + assertEquals("name", tbl2.getSd().getBucketCols().get(0)); + assertTrue("Partition key list is not empty", + (tbl2.getPartitionKeys() == null) + || (tbl2.getPartitionKeys().size() == 0)); + + //test get_table_objects_by_name functionality + ArrayList<String> tableNames = new ArrayList<String>(); + tableNames.add(tblName2); + tableNames.add(tblName); + tableNames.add(tblName2); + List<Table> foundTables = client.getTableObjectsByName(dbName, tableNames); + + assertEquals(2, foundTables.size()); + for (Table t: foundTables) { + if (t.getTableName().equals(tblName2)) { + assertEquals(t.getSd().getLocation(), tbl2.getSd().getLocation()); + } else { + assertEquals(t.getTableName(), tblName); + assertEquals(t.getSd().getLocation(), tbl.getSd().getLocation()); + } + assertEquals(t.getSd().getCols().size(), typ1.getFields().size()); + assertEquals(t.getSd().isCompressed(), false); + assertEquals(foundTables.get(0).getSd().getNumBuckets(), 1); + assertNotNull(t.getSd().getSerdeInfo()); + assertEquals(t.getDbName(), dbName); + } + + tableNames.add(1, "table_that_doesnt_exist"); + foundTables = client.getTableObjectsByName(dbName, tableNames); + assertEquals(foundTables.size(), 2); + + InvalidOperationException ioe = null; + try { + foundTables = client.getTableObjectsByName(dbName, null); + } catch (InvalidOperationException e) { + ioe = e; + } + assertNotNull(ioe); + assertTrue("Table not found", ioe.getMessage().contains("null tables")); + + UnknownDBException udbe = null; + try { + foundTables = client.getTableObjectsByName("db_that_doesnt_exist", tableNames); + } catch (UnknownDBException e) { + udbe = e; + } + assertNotNull(udbe); + assertTrue("DB not found", udbe.getMessage().contains("not find database db_that_doesnt_exist")); + + udbe = null; + try { + foundTables = client.getTableObjectsByName("", tableNames); + } catch (UnknownDBException e) { + udbe = e; + } + assertNotNull(udbe); + assertTrue("DB not found", udbe.getMessage().contains("is null or empty")); + + FileSystem fs = FileSystem.get((new Path(tbl.getSd().getLocation())).toUri(), hiveConf); + client.dropTable(dbName, tblName); + assertFalse(fs.exists(new Path(tbl.getSd().getLocation()))); + + client.dropTable(dbName, tblName2); + assertTrue(fs.exists(new Path(tbl2.getSd().getLocation()))); + + client.dropType(typeName); + client.dropDatabase(dbName); + } catch (Exception e) { + System.err.println(StringUtils.stringifyException(e)); + System.err.println("testSimpleTable() failed."); + throw e; + } + } + + // Tests that in the absence of stats for partitions, and/or absence of columns + // to get stats for, the metastore does not break. See HIVE-12083 for motivation. + public void testStatsFastTrivial() throws Throwable { + String dbName = "tstatsfast"; + String tblName = "t1"; + String tblOwner = "statstester"; + String typeName = "Person"; + int lastAccessed = 12083; + + cleanUp(dbName,tblName,typeName); + + List<List<String>> values = new ArrayList<List<String>>(); + values.add(makeVals("2008-07-01 14:13:12", "14")); + values.add(makeVals("2008-07-01 14:13:12", "15")); + values.add(makeVals("2008-07-02 14:13:12", "15")); + values.add(makeVals("2008-07-03 14:13:12", "151")); + + createMultiPartitionTableSchema(dbName, tblName, typeName, values); + + List<String> emptyColNames = new ArrayList<String>(); + List<String> emptyPartNames = new ArrayList<String>(); + + List<String> colNames = new ArrayList<String>(); + colNames.add("name"); + colNames.add("income"); + List<String> partNames = client.listPartitionNames(dbName,tblName,(short)-1); + + assertEquals(0,emptyColNames.size()); + assertEquals(0,emptyPartNames.size()); + assertEquals(2,colNames.size()); + assertEquals(4,partNames.size()); + + // Test for both colNames and partNames being empty: + AggrStats aggrStatsEmpty = client.getAggrColStatsFor(dbName,tblName,emptyColNames,emptyPartNames); + assertNotNull(aggrStatsEmpty); // short-circuited on client-side, verifying that it's an empty object, not null + assertEquals(0,aggrStatsEmpty.getPartsFound()); + assertNotNull(aggrStatsEmpty.getColStats()); + assert(aggrStatsEmpty.getColStats().isEmpty()); + + // Test for only colNames being empty + AggrStats aggrStatsOnlyParts = client.getAggrColStatsFor(dbName,tblName,emptyColNames,partNames); + assertNotNull(aggrStatsOnlyParts); // short-circuited on client-side, verifying that it's an empty object, not null + assertEquals(0,aggrStatsOnlyParts.getPartsFound()); + assertNotNull(aggrStatsOnlyParts.getColStats()); + assert(aggrStatsOnlyParts.getColStats().isEmpty()); + + // Test for only partNames being empty + AggrStats aggrStatsOnlyCols = client.getAggrColStatsFor(dbName,tblName,colNames,emptyPartNames); + assertNotNull(aggrStatsOnlyCols); // short-circuited on client-side, verifying that it's an empty object, not null + assertEquals(0,aggrStatsOnlyCols.getPartsFound()); + assertNotNull(aggrStatsOnlyCols.getColStats()); + assert(aggrStatsOnlyCols.getColStats().isEmpty()); + + // Test for valid values for both. + AggrStats aggrStatsFull = client.getAggrColStatsFor(dbName,tblName,colNames,partNames); + assertNotNull(aggrStatsFull); + assertEquals(0,aggrStatsFull.getPartsFound()); // would still be empty, because no stats are actually populated. + assertNotNull(aggrStatsFull.getColStats()); + assert(aggrStatsFull.getColStats().isEmpty()); + + } + + public void testColumnStatistics() throws Throwable { + + String dbName = "columnstatstestdb"; + String tblName = "tbl"; + String typeName = "Person"; + String tblOwner = "testowner"; + int lastAccessed = 6796; + + try { + cleanUp(dbName, tblName, typeName); + Database db = new Database(); + db.setName(dbName); + client.createDatabase(db); + createTableForTestFilter(dbName,tblName, tblOwner, lastAccessed, true); + + // Create a ColumnStatistics Obj + String[] colName = new String[]{"income", "name"}; + double lowValue = 50000.21; + double highValue = 1200000.4525; + long numNulls = 3; + long numDVs = 22; + double avgColLen = 50.30; + long maxColLen = 102; + String[] colType = new String[] {"double", "string"}; + boolean isTblLevel = true; + String partName = null; + List<ColumnStatisticsObj> statsObjs = new ArrayList<ColumnStatisticsObj>(); + + ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(); + statsDesc.setDbName(dbName); + statsDesc.setTableName(tblName); + statsDesc.setIsTblLevel(isTblLevel); + statsDesc.setPartName(partName); + + ColumnStatisticsObj statsObj = new ColumnStatisticsObj(); + statsObj.setColName(colName[0]); + statsObj.setColType(colType[0]); + + ColumnStatisticsData statsData = new ColumnStatisticsData(); + DoubleColumnStatsData numericStats = new DoubleColumnStatsData(); + statsData.setDoubleStats(numericStats); + + statsData.getDoubleStats().setHighValue(highValue); + statsData.getDoubleStats().setLowValue(lowValue); + statsData.getDoubleStats().setNumDVs(numDVs); + statsData.getDoubleStats().setNumNulls(numNulls); + + statsObj.setStatsData(statsData); + statsObjs.add(statsObj); + + statsObj = new ColumnStatisticsObj(); + statsObj.setColName(colName[1]); + statsObj.setColType(colType[1]); + + statsData = new ColumnStatisticsData(); + StringColumnStatsData stringStats = new StringColumnStatsData(); + statsData.setStringStats(stringStats); + statsData.getStringStats().setAvgColLen(avgColLen); + statsData.getStringStats().setMaxColLen(maxColLen); + statsData.getStringStats().setNumDVs(numDVs); + statsData.getStringStats().setNumNulls(numNulls); + + statsObj.setStatsData(statsData); + statsObjs.add(statsObj); + + ColumnStatistics colStats = new ColumnStatistics(); + colStats.setStatsDesc(statsDesc); + colStats.setStatsObj(statsObjs); + + // write stats objs persistently + client.updateTableColumnStatistics(colStats); + + // retrieve the stats obj that was just written + ColumnStatisticsObj colStats2 = client.getTableColumnStatistics( + dbName, tblName, Lists.newArrayList(colName[0])).get(0); + + // compare stats obj to ensure what we get is what we wrote + assertNotNull(colStats2); + assertEquals(colStats2.getColName(), colName[0]); + assertEquals(colStats2.getStatsData().getDoubleStats().getLowValue(), lowValue); + assertEquals(colStats2.getStatsData().getDoubleStats().getHighValue(), highValue); + assertEquals(colStats2.getStatsData().getDoubleStats().getNumNulls(), numNulls); + assertEquals(colStats2.getStatsData().getDoubleStats().getNumDVs(), numDVs); + + // test delete column stats; if no col name is passed all column stats associated with the + // table is deleted + boolean status = client.deleteTableColumnStatistics(dbName, tblName, null); + assertTrue(status); + // try to query stats for a column for which stats doesn't exist + assertTrue(client.getTableColumnStatistics( + dbName, tblName, Lists.newArrayList(colName[1])).isEmpty()); + + colStats.setStatsDesc(statsDesc); + colStats.setStatsObj(statsObjs); + + // update table level column stats + client.updateTableColumnStatistics(colStats); + + // query column stats for column whose stats were updated in the previous call + colStats2 = client.getTableColumnStatistics( + dbName, tblName, Lists.newArrayList(colName[0])).get(0); + + // partition level column statistics test + // create a table with multiple partitions + cleanUp(dbName, tblName, typeName); + + List<List<String>> values = new ArrayList<List<String>>(); + values.add(makeVals("2008-07-01 14:13:12", "14")); + values.add(makeVals("2008-07-01 14:13:12", "15")); + values.add(makeVals("2008-07-02 14:13:12", "15")); + values.add(makeVals("2008-07-03 14:13:12", "151")); + + createMultiPartitionTableSchema(dbName, tblName, typeName, values); + + List<String> partitions = client.listPartitionNames(dbName, tblName, (short)-1); + + partName = partitions.get(0); + isTblLevel = false; + + // create a new columnstatistics desc to represent partition level column stats + statsDesc = new ColumnStatisticsDesc(); + statsDesc.setDbName(dbName); + statsDesc.setTableName(tblName); + statsDesc.setPartName(partName); + statsDesc.setIsTblLevel(isTblLevel); + + colStats = new ColumnStatistics(); + colStats.setStatsDesc(statsDesc); + colStats.setStatsObj(statsObjs); + + client.updatePartitionColumnStatistics(colStats); + + colStats2 = client.getPartitionColumnStatistics(dbName, tblName, + Lists.newArrayList(partName), Lists.newArrayList(colName[1])).get(partName).get(0); + + // compare stats obj to ensure what we get is what we wrote + assertNotNull(colStats2); + assertEquals(colStats.getStatsDesc().getPartName(), partName); + assertEquals(colStats2.getColName(), colName[1]); + assertEquals(colStats2.getStatsData().getStringStats().getMaxColLen(), maxColLen); + assertEquals(colStats2.getStatsData().getStringStats().getAvgColLen(), avgColLen); + assertEquals(colStats2.getStatsData().getStringStats().getNumNulls(), numNulls); + assertEquals(colStats2.getStatsData().getStringStats().getNumDVs(), numDVs); + + // test stats deletion at partition level + client.deletePartitionColumnStatistics(dbName, tblName, partName, colName[1]); + + colStats2 = client.getPartitionColumnStatistics(dbName, tblName, + Lists.newArrayList(partName), Lists.newArrayList(colName[0])).get(partName).get(0); + + // test get stats on a column for which stats doesn't exist + assertTrue(client.getPartitionColumnStatistics(dbName, tblName, + Lists.newArrayList(partName), Lists.newArrayList(colName[1])).isEmpty()); + } catch (Exception e) { + System.err.println(StringUtils.stringifyException(e)); + System.err.println("testColumnStatistics() failed."); + throw e; + } finally { + cleanUp(dbName, tblName, typeName); + } + } + + public void testAlterTable() throws Exception { + String dbName = "alterdb"; + String invTblName = "alter-tbl"; + String tblName = "altertbl"; + + try { + client.dropTable(dbName, tblName); + silentDropDatabase(dbName); + + Database db = new Database(); + db.setName(dbName); + client.createDatabase(db); + + ArrayList<FieldSchema> invCols = new ArrayList<FieldSchema>(2); + invCols.add(new FieldSchema("n-ame", serdeConstants.STRING_TYPE_NAME, "")); + invCols.add(new FieldSchema("in.come", serdeConstants.INT_TYPE_NAME, "")); + + Table tbl = new Table(); + tbl.setDbName(dbName); + tbl.setTableName(invTblName); + StorageDescriptor sd = new StorageDescriptor(); + tbl.setSd(sd); + sd.setCols(invCols); + sd.setCompressed(false); + sd.setNumBuckets(1); + sd.setParameters(new HashMap<String, String>()); + sd.getParameters().put("test_param_1", "Use this for comments etc"); + sd.setBucketCols(new ArrayList<String>(2)); + sd.getBucketCols().add("name"); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setName(tbl.getTableName()); + sd.getSerdeInfo().setParameters(new HashMap<String, String>()); + sd.getSerdeInfo().getParameters().put( + org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1"); + sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName()); + sd.setInputFormat(HiveInputFormat.class.getName()); + sd.setOutputFormat(HiveOutputFormat.class.getName()); + + boolean failed = false; + try { + client.createTable(tbl); + } catch (InvalidObjectException ex) { + failed = true; + } + if (!failed) { + assertTrue("Able to create table with invalid name: " + invTblName, + false); + } + + // create an invalid table which has wrong column type + ArrayList<FieldSchema> invColsInvType = new ArrayList<FieldSchema>(2); + invColsInvType.add(new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, "")); + invColsInvType.add(new FieldSchema("income", "xyz", "")); + tbl.setTableName(tblName); + tbl.getSd().setCols(invColsInvType); + boolean failChecker = false; + try { + client.createTable(tbl); + } catch (InvalidObjectException ex) { + failChecker = true; + } + if (!failChecker) { + assertTrue("Able to create table with invalid column type: " + invTblName, + false); + } + + ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(2); + cols.add(new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, "")); + cols.add(new FieldSchema("income", serdeConstants.INT_TYPE_NAME, "")); + + // create a valid table + tbl.setTableName(tblName); + tbl.getSd().setCols(cols); + client.createTable(tbl); + + if (isThriftClient) { + tbl = client.getTable(tbl.getDbName(), tbl.getTableName()); + } + + // now try to invalid alter table + Table tbl2 = client.getTable(dbName, tblName); + failed = false; + try { + tbl2.setTableName(invTblName); + tbl2.getSd().setCols(invCols); + client.alter_table(dbName, tblName, tbl2); + } catch (InvalidOperationException ex) { + failed = true; + } + if (!failed) { + assertTrue("Able to rename table with invalid name: " + invTblName, + false); + } + + //try an invalid alter table with partition key name + Table tbl_pk = client.getTable(tbl.getDbName(), tbl.getTableName()); + List<FieldSchema> partitionKeys = tbl_pk.getPartitionKeys(); + for (FieldSchema fs : partitionKeys) { + fs.setName("invalid_to_change_name"); + fs.setComment("can_change_comment"); + } + tbl_pk.setPartitionKeys(partitionKeys); + try { + client.alter_table(dbName, tblName, tbl_pk); + } catch (InvalidOperationException ex) { + failed = true; + } + assertTrue("Should not have succeeded in altering partition key name", failed); + + //try a valid alter table partition key comment + failed = false; + tbl_pk = client.getTable(tbl.getDbName(), tbl.getTableName()); + partitionKeys = tbl_pk.getPartitionKeys(); + for (FieldSchema fs : partitionKeys) { + fs.setComment("can_change_comment"); + } + tbl_pk.setPartitionKeys(partitionKeys); + try { + client.alter_table(dbName, tblName, tbl_pk); + } catch (InvalidOperationException ex) { + failed = true; + } + assertFalse("Should not have failed alter table partition comment", failed); + Table newT = client.getTable(tbl.getDbName(), tbl.getTableName()); + assertEquals(partitionKeys, newT.getPartitionKeys()); + + // try a valid alter table + tbl2.setTableName(tblName + "_renamed"); + tbl2.getSd().setCols(cols); + tbl2.getSd().setNumBuckets(32); + client.alter_table(dbName, tblName, tbl2); + Table tbl3 = client.getTable(dbName, tbl2.getTableName()); + assertEquals("Alter table didn't succeed. Num buckets is different ", + tbl2.getSd().getNumBuckets(), tbl3.getSd().getNumBuckets()); + // check that data has moved + FileSystem fs = FileSystem.get((new Path(tbl.getSd().getLocation())).toUri(), hiveConf); + assertFalse("old table location still exists", fs.exists(new Path(tbl + .getSd().getLocation()))); + assertTrue("data did not move to new location", fs.exists(new Path(tbl3 + .getSd().getLocation()))); + + if (!isThriftClient) { + assertEquals("alter table didn't move data correct location", tbl3 + .getSd().getLocation(), tbl2.getSd().getLocation()); + } + + // alter table with invalid column type + tbl_pk.getSd().setCols(invColsInvType); + failed = false; + try { + client.alter_table(dbName, tbl2.getTableName(), tbl_pk); + } catch (InvalidOperationException ex) { + failed = true; + } + assertTrue("Should not have succeeded in altering column", failed); + + } catch (Exception e) { + System.err.println(StringUtils.stringifyException(e)); + System.err.println("testSimpleTable() failed."); + throw e; + } finally { + silentDropDatabase(dbName); + } + } + + public void testComplexTable() throws Exception { + + String dbName = "compdb"; + String tblName = "comptbl"; + String typeName = "Person"; + + try { + client.dropTable(dbName, tblName); + silentDropDatabase(dbName); + Database db = new Database(); + db.setName(dbName); + client.createDatabase(db); + + client.dropType(typeName); + Type typ1 = new Type(); + typ1.setName(typeName); + typ1.setFields(new ArrayList<FieldSchema>(2)); + typ1.getFields().add( + new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, "")); + typ1.getFields().add( + new FieldSchema("income", serdeConstants.INT_TYPE_NAME, "")); + client.createType(typ1); + + Table tbl = new Table(); + tbl.setDbName(dbName); + tbl.setTableName(tblName); + StorageDescriptor sd = new StorageDescriptor(); + tbl.setSd(sd); + sd.setCols(typ1.getFields()); + sd.setCompressed(false); + sd.setNumBuckets(1); + sd.setParameters(new HashMap<String, String>()); + sd.getParameters().put("test_param_1", "Use this for comments etc"); + sd.setBucketCols(new ArrayList<String>(2)); + sd.getBucketCols().add("name"); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setName(tbl.getTableName()); + sd.getSerdeInfo().setParameters(new HashMap<String, String>()); + sd.getSerdeInfo().getParameters().put( + org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "9"); + sd.getSerdeInfo().setSerializationLib( + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); + sd.setInputFormat(HiveInputFormat.class.getName()); + sd.setOutputFormat(HiveOutputFormat.class.getName()); + + tbl.setPartitionKeys(new ArrayList<FieldSchema>(2)); + tbl.getPartitionKeys().add( + new FieldSchema("ds", + org.apache.hadoop.hive.serde.serdeConstants.DATE_TYPE_NAME, "")); + tbl.getPartitionKeys().add( + new FieldSchema("hr", + org.apache.hadoop.hive.serde.serdeConstants.INT_TYPE_NAME, "")); + + client.createTable(tbl); + + Table tbl2 = client.getTable(dbName, tblName); + assertEquals(tbl2.getDbName(), dbName); + assertEquals(tbl2.getTableName(), tblName); + assertEquals(tbl2.getSd().getCols().size(), typ1.getFields().size()); + assertFalse(tbl2.getSd().isCompressed()); + assertFalse(tbl2.getSd().isStoredAsSubDirectories()); + assertEquals(tbl2.getSd().getNumBuckets(), 1); + + assertEquals("Use this for comments etc", tbl2.getSd().getParameters() + .get("test_param_1")); + assertEquals("name", tbl2.getSd().getBucketCols().get(0)); + + assertNotNull(tbl2.getPartitionKeys()); + assertEquals(2, tbl2.getPartitionKeys().size()); + assertEquals(serdeConstants.DATE_TYPE_NAME, tbl2.getPartitionKeys().get(0) + .getType()); + assertEquals(serdeConstants.INT_TYPE_NAME, tbl2.getPartitionKeys().get(1) + .getType()); + assertEquals("ds", tbl2.getPartitionKeys().get(0).getName()); + assertEquals("hr", tbl2.getPartitionKeys().get(1).getName()); + + List<FieldSchema> fieldSchemas = client.getFields(dbName, tblName); + assertNotNull(fieldSchemas); + assertEquals(fieldSchemas.size(), tbl.getSd().getCols().size()); + for (FieldSchema fs : tbl.getSd().getCols()) { + assertTrue(fieldSchemas.contains(fs)); + } + + List<FieldSchema> fieldSchemasFull = client.getSchema(dbName, tblName); + assertNotNull(fieldSchemasFull); + assertEquals(fieldSchemasFull.size(), tbl.getSd().getCols().size() + + tbl.getPartitionKeys().size()); + for (FieldSchema fs : tbl.getSd().getCols()) { + assertTrue(fieldSchemasFull.contains(fs)); + } + for (FieldSchema fs : tbl.getPartitionKeys()) { + assertTrue(fieldSchemasFull.contains(fs)); + } + } catch (Exception e) { + System.err.println(StringUtils.stringifyException(e)); + System.err.println("testComplexTable() failed."); + throw e; + } finally { + client.dropTable(dbName, tblName); + boolean ret = client.dropType(typeName); + assertTrue("Unable to drop type " + typeName, ret); + client.dropDatabase(dbName); + } + } + + public void testTableDatabase() throws Exception { + String dbName = "testDb"; + String tblName_1 = "testTbl_1"; + String tblName_2 = "testTbl_2"; + + try { + silentDropDatabase(dbName); + + Database db = new Database(); + db.setName(dbName); + String dbLocation = + HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTOREWAREHOUSE) + "_testDB_table_create_"; + db.setLocationUri(dbLocation); + client.createDatabase(db); + db = client.getDatabase(dbName); + + Table tbl = new Table(); + tbl.setDbName(dbName); + tbl.setTableName(tblName_1); + + ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(2); + cols.add(new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, "")); + cols.add(new FieldSchema("income", serdeConstants.INT_TYPE_NAME, "")); + + StorageDescriptor sd = new StorageDescriptor(); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setName(tbl.getTableName()); + sd.getSerdeInfo().setParameters(new HashMap<String, String>()); + sd.setParameters(new HashMap<String, String>()); + sd.getSerdeInfo().getParameters().put( + org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "9"); + sd.getSerdeInfo().setSerializationLib( + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); + sd.setInputFormat(HiveInputFormat.class.getName()); + sd.setOutputFormat(HiveOutputFormat.class.getName()); + + tbl.setSd(sd); + tbl.getSd().setCols(cols); + client.createTable(tbl); + tbl = client.getTable(dbName, tblName_1); + + Path path = new Path(tbl.getSd().getLocation()); + System.err.println("Table's location " + path + ", Database's location " + db.getLocationUri()); + assertEquals("Table location is not a subset of the database location", + path.getParent().toString(), db.getLocationUri()); + + } catch (Exception e) { + System.err.println(StringUtils.stringifyException(e)); + System.err.println("testTableDatabase() failed."); + throw e; + } finally { + silentDropDatabase(dbName); + } + } + + + public void testGetConfigValue() { + + String val = "value"; + + if (!isThriftClient) { + try { + assertEquals(client.getConfigValue("hive.key1", val), "value1"); + assertEquals(client.getConfigValue("hive.key2", val), "http://www.example.com"); + assertEquals(client.getConfigValue("hive.key3", val), ""); + assertEquals(client.getConfigValue("hive.key4", val), "0"); + assertEquals(client.getConfigValue("hive.key5", val), val); + assertEquals(client.getConfigValue(null, val), val); + } catch (ConfigValSecurityException e) { + e.printStackTrace(); + assert (false); + } catch (TException e) { + e.printStackTrace(); + assert (false); + } + } + + boolean threwException = false; + try { + // Attempting to get the password should throw an exception + client.getConfigValue("javax.jdo.option.ConnectionPassword", "password"); + } catch (ConfigValSecurityException e) { + threwException = true; + } catch (TException e) { + e.printStackTrace(); + assert (false); + } + assert (threwException); + } + + private static void adjust(HiveMetaStoreClient client, Partition part, + String dbName, String tblName) + throws NoSuchObjectException, MetaException, TException { + Partition part_get = client.getPartition(dbName, tblName, part.getValues()); + part.setCreateTime(part_get.getCreateTime()); + part.putToParameters(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.DDL_TIME, Long.toString(part_get.getCreateTime())); + } + + private static void silentDropDatabase(String dbName) throws MetaException, TException { + try { + for (String tableName : client.getTables(dbName, "*")) { + client.dropTable(dbName, tableName); + } + client.dropDatabase(dbName); + } catch (NoSuchObjectException e) { + } catch (InvalidOperationException e) { + } + } + + /** + * Tests for list partition by filter functionality. + * @throws Exception + */ + + public void testPartitionFilter() throws Exception { + String dbName = "filterdb"; + String tblName = "filtertbl"; + + silentDropDatabase(dbName); + + Database db = new Database(); + db.setName(dbName); + client.createDatabase(db); + + ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(2); + cols.add(new FieldSchema("c1", serdeConstants.STRING_TYPE_NAME, "")); + cols.add(new FieldSchema("c2", serdeConstants.INT_TYPE_NAME, "")); + + ArrayList<FieldSchema> partCols = new ArrayList<FieldSchema>(3); + partCols.add(new FieldSchema("p1", serdeConstants.STRING_TYPE_NAME, "")); + partCols.add(new FieldSchema("p2", serdeConstants.STRING_TYPE_NAME, "")); + partCols.add(new FieldSchema("p3", serdeConstants.INT_TYPE_NAME, "")); + + Table tbl = new Table(); + tbl.setDbName(dbName); + tbl.setTableName(tblName); + StorageDescriptor sd = new StorageDescriptor(); + tbl.setSd(sd); + sd.setCols(cols); + sd.setCompressed(false); + sd.setNumBuckets(1); + sd.setParameters(new HashMap<String, String>()); + sd.setBucketCols(new ArrayList<String>()); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setName(tbl.getTableName()); + sd.getSerdeInfo().setParameters(new HashMap<String, String>()); + sd.getSerdeInfo().getParameters() + .put(serdeConstants.SERIALIZATION_FORMAT, "1"); + sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName()); + sd.setInputFormat(HiveInputFormat.class.getName()); + sd.setOutputFormat(HiveOutputFormat.class.getName()); + sd.setSortCols(new ArrayList<Order>()); + + tbl.setPartitionKeys(partCols); + client.createTable(tbl); + + tbl = client.getTable(dbName, tblName); + + add_partition(client, tbl, Lists.newArrayList("p11", "p21", "31"), "part1"); + add_partition(client, tbl, Lists.newArrayList("p11", "p22", "32"), "part2"); + add_partition(client, tbl, Lists.newArrayList("p12", "p21", "31"), "part3"); + add_partition(client, tbl, Lists.newArrayList("p12", "p23", "32"), "part4"); + add_partition(client, tbl, Lists.newArrayList("p13", "p24", "31"), "part5"); + add_partition(client, tbl, Lists.newArrayList("p13", "p25", "-33"), "part6"); + + // Test equals operator for strings and integers. + checkFilter(client, dbName, tblName, "p1 = \"p11\"", 2); + checkFilter(client, dbName, tblName, "p1 = \"p12\"", 2); + checkFilter(client, dbName, tblName, "p2 = \"p21\"", 2); + checkFilter(client, dbName, tblName, "p2 = \"p23\"", 1); + checkFilter(client, dbName, tblName, "p3 = 31", 3); + checkFilter(client, dbName, tblName, "p3 = 33", 0); + checkFilter(client, dbName, tblName, "p3 = -33", 1); + checkFilter(client, dbName, tblName, "p1 = \"p11\" and p2=\"p22\"", 1); + checkFilter(client, dbName, tblName, "p1 = \"p11\" or p2=\"p23\"", 3); + checkFilter(client, dbName, tblName, "p1 = \"p11\" or p1=\"p12\"", 4); + checkFilter(client, dbName, tblName, "p1 = \"p11\" or p1=\"p12\"", 4); + checkFilter(client, dbName, tblName, "p1 = \"p11\" or p1=\"p12\"", 4); + checkFilter(client, dbName, tblName, "p1 = \"p11\" and p3 = 31", 1); + checkFilter(client, dbName, tblName, "p3 = -33 or p1 = \"p12\"", 3); + + // Test not-equals operator for strings and integers. + checkFilter(client, dbName, tblName, "p1 != \"p11\"", 4); + checkFilter(client, dbName, tblName, "p2 != \"p23\"", 5); + checkFilter(client, dbName, tblName, "p2 != \"p33\"", 6); + checkFilter(client, dbName, tblName, "p3 != 32", 4); + checkFilter(client, dbName, tblName, "p3 != 8589934592", 6); + checkFilter(client, dbName, tblName, "p1 != \"p11\" and p1 != \"p12\"", 2); + checkFilter(client, dbName, tblName, "p1 != \"p11\" and p2 != \"p22\"", 4); + checkFilter(client, dbName, tblName, "p1 != \"p11\" or p2 != \"p22\"", 5); + checkFilter(client, dbName, tblName, "p1 != \"p12\" and p2 != \"p25\"", 3); + checkFilter(client, dbName, tblName, "p1 != \"p12\" or p2 != \"p25\"", 6); + checkFilter(client, dbName, tblName, "p3 != -33 or p1 != \"p13\"", 5); + checkFilter(client, dbName, tblName, "p1 != \"p11\" and p3 = 31", 2); + checkFilter(client, dbName, tblName, "p3 != 31 and p1 = \"p12\"", 1); + + // Test reverse order. + checkFilter(client, dbName, tblName, "31 != p3 and p1 = \"p12\"", 1); + checkFilter(client, dbName, tblName, "\"p23\" = p2", 1); + + // Test and/or more... + checkFilter(client, dbName, tblName, + "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\")", 3); + checkFilter(client, dbName, tblName, + "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\") Or " + + "(p1=\"p13\" aNd p2=\"p24\")", 4); + //test for and or precedence + checkFilter(client, dbName, tblName, + "p1=\"p12\" and (p2=\"p27\" Or p2=\"p21\")", 1); + checkFilter(client, dbName, tblName, + "p1=\"p12\" and p2=\"p27\" Or p2=\"p21\"", 2); + + // Test gt/lt/lte/gte/like for strings. + checkFilter(client, dbName, tblName, "p1 > \"p12\"", 2); + checkFilter(client, dbName, tblName, "p1 >= \"p12\"", 4); + checkFilter(client, dbName, tblName, "p1 < \"p12\"", 2); + checkFilter(client, dbName, tblName, "p1 <= \"p12\"", 4); + checkFilter(client, dbName, tblName, "p1 like \"p1.*\"", 6); + checkFilter(client, dbName, tblName, "p2 like \"p.*3\"", 1); + + // Test gt/lt/lte/gte for numbers. + checkFilter(client, dbName, tblName, "p3 < 0", 1); + checkFilter(client, dbName, tblName, "p3 >= -33", 6); + checkFilter(client, dbName, tblName, "p3 > -33", 5); + checkFilter(client, dbName, tblName, "p3 > 31 and p3 < 32", 0); + checkFilter(client, dbName, tblName, "p3 > 31 or p3 < 31", 3); + checkFilter(client, dbName, tblName, "p3 > 30 or p3 < 30", 6); + checkFilter(client, dbName, tblName, "p3 >= 31 or p3 < -32", 6); + checkFilter(client, dbName, tblName, "p3 >= 32", 2); + checkFilter(client, dbName, tblName, "p3 > 32", 0); + + // Test between + checkFilter(client, dbName, tblName, "p1 between \"p11\" and \"p12\"", 4); + checkFilter(client, dbName, tblName, "p1 not between \"p11\" and \"p12\"", 2); + checkFilter(client, dbName, tblName, "p3 not between 0 and 2", 6); + checkFilter(client, dbName, tblName, "p3 between 31 and 32", 5); + checkFilter(client, dbName, tblName, "p3 between 32 and 31", 0); + checkFilter(client, dbName, tblName, "p3 between -32 and 34 and p3 not between 31 and 32", 0); + checkFilter(client, dbName, tblName, "p3 between 1 and 3 or p3 not between 1 and 3", 6); + checkFilter(client, dbName, tblName, + "p3 between 31 and 32 and p1 between \"p12\" and \"p14\"", 3); + + //Test for setting the maximum partition count + List<Partition> partitions = client.listPartitionsByFilter(dbName, + tblName, "p1 >= \"p12\"", (short) 2); + assertEquals("User specified row limit for partitions", + 2, partitions.size()); + + //Negative tests + Exception me = null; + try { + client.listPartitionsByFilter(dbName, + tblName, "p3 >= \"p12\"", (short) -1); + } catch(MetaException e) { + me = e; + } + assertNotNull(me); + assertTrue("Filter on int partition key", me.getMessage().contains( + "Filtering is supported only on partition keys of type string")); + + me = null; + try { + client.listPartitionsByFilter(dbName, + tblName, "c1 >=
<TRUNCATED>