http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/main/java/org/hadoop/hive/jdbc/SSLTestUtils.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/main/java/org/hadoop/hive/jdbc/SSLTestUtils.java b/itests/hive-unit/src/main/java/org/hadoop/hive/jdbc/SSLTestUtils.java new file mode 100644 index 0000000..6cbcf8c --- /dev/null +++ b/itests/hive-unit/src/main/java/org/hadoop/hive/jdbc/SSLTestUtils.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.hadoop.hive.jdbc; + +import java.io.File; +import java.net.URLEncoder; +import java.sql.Connection; +import java.sql.Statement; +import java.util.Map; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; + + +public class SSLTestUtils { + + private static final String LOCALHOST_KEY_STORE_NAME = "keystore.jks"; + private static final String TRUST_STORE_NAME = "truststore.jks"; + private static final String KEY_STORE_TRUST_STORE_PASSWORD = "HiveJdbc"; + private static final String HS2_BINARY_MODE = "binary"; + private static final String HS2_HTTP_MODE = "http"; + private static final String HS2_HTTP_ENDPOINT = "cliservice"; + private static final String HS2_BINARY_AUTH_MODE = "NONE"; + + private static final HiveConf conf = new HiveConf(); + private static final String dataFileDir = !System.getProperty("test.data.files", "").isEmpty() ? System.getProperty( + "test.data.files") : conf.get("test.data.files").replace('\\', '/').replace("c:", ""); + + public static final String SSL_CONN_PARAMS = "ssl=true;sslTrustStore=" + + URLEncoder.encode(dataFileDir + File.separator + TRUST_STORE_NAME) + ";trustStorePassword=" + + KEY_STORE_TRUST_STORE_PASSWORD; + + public static void setSslConfOverlay(Map<String, String> confOverlay) { + confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_USE_SSL.varname, "true"); + confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname, + dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME); + confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname, + KEY_STORE_TRUST_STORE_PASSWORD); + } + + public static void setMetastoreSslConf(HiveConf conf) { + conf.setBoolVar(HiveConf.ConfVars.HIVE_METASTORE_USE_SSL, true); + conf.setVar(HiveConf.ConfVars.HIVE_METASTORE_SSL_KEYSTORE_PATH, + dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME); + conf.setVar(HiveConf.ConfVars.HIVE_METASTORE_SSL_KEYSTORE_PASSWORD, + KEY_STORE_TRUST_STORE_PASSWORD); + conf.setVar(HiveConf.ConfVars.HIVE_METASTORE_SSL_TRUSTSTORE_PATH, + dataFileDir + File.separator + TRUST_STORE_NAME); + conf.setVar(HiveConf.ConfVars.HIVE_METASTORE_SSL_TRUSTSTORE_PASSWORD, + KEY_STORE_TRUST_STORE_PASSWORD); + } + + public static void clearSslConfOverlay(Map<String, String> confOverlay) { + confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_USE_SSL.varname, "false"); + } + + public static void setHttpConfOverlay(Map<String, String> confOverlay) { + confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_HTTP_MODE); + confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, HS2_HTTP_ENDPOINT); + confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true"); + } + + public static void setBinaryConfOverlay(Map<String, String> confOverlay) { + confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_BINARY_MODE); + confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION.varname, HS2_BINARY_AUTH_MODE); + confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true"); + } + + public static void setupTestTableWithData(String tableName, Path dataFilePath, + Connection hs2Conn) throws Exception { + Statement stmt = hs2Conn.createStatement(); + stmt.execute("set hive.support.concurrency = false"); + + stmt.execute("drop table if exists " + tableName); + stmt.execute("create table " + tableName + + " (under_col int comment 'the under column', value string)"); + + // load data + stmt.execute("load data local inpath '" + + dataFilePath.toString() + "' into table " + tableName); + stmt.close(); + } + + public static String getDataFileDir() { + return dataFileDir; + } +}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestEmbeddedHiveMetaStore.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestEmbeddedHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestEmbeddedHiveMetaStore.java index c6a906a..bc00d11 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestEmbeddedHiveMetaStore.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestEmbeddedHiveMetaStore.java @@ -26,8 +26,6 @@ public class TestEmbeddedHiveMetaStore extends TestHiveMetaStore { @Override protected void setUp() throws Exception { super.setUp(); - hiveConf.setBoolean( - HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS.varname, true); warehouse = new Warehouse(hiveConf); try { client = new HiveMetaStoreClient(hiveConf); http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index af125c3..b95c25c 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -171,14 +171,6 @@ public abstract class TestHiveMetaStore extends TestCase { db = client.getDatabase(dbName); Path dbPath = new Path(db.getLocationUri()); FileSystem fs = FileSystem.get(dbPath.toUri(), hiveConf); - boolean inheritPerms = hiveConf.getBoolVar( - HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS); - FsPermission dbPermission = fs.getFileStatus(dbPath).getPermission(); - if (inheritPerms) { - //Set different perms for the database dir for further tests - dbPermission = new FsPermission((short)488); - fs.setPermission(dbPath, dbPermission); - } client.dropType(typeName); Type typ1 = new Type(); @@ -239,9 +231,6 @@ public abstract class TestHiveMetaStore extends TestCase { tbl = client.getTable(dbName, tblName); } - assertEquals(dbPermission, fs.getFileStatus(new Path(tbl.getSd().getLocation())) - .getPermission()); - Partition part = makePartitionObject(dbName, tblName, vals, tbl, "/part1"); Partition part2 = makePartitionObject(dbName, tblName, vals2, tbl, "/part2"); Partition part3 = makePartitionObject(dbName, tblName, vals3, tbl, "/part3"); @@ -259,20 +248,12 @@ public abstract class TestHiveMetaStore extends TestCase { assertTrue("getPartition() should have thrown NoSuchObjectException", exceptionThrown); Partition retp = client.add_partition(part); assertNotNull("Unable to create partition " + part, retp); - assertEquals(dbPermission, fs.getFileStatus(new Path(retp.getSd().getLocation())) - .getPermission()); Partition retp2 = client.add_partition(part2); assertNotNull("Unable to create partition " + part2, retp2); - assertEquals(dbPermission, fs.getFileStatus(new Path(retp2.getSd().getLocation())) - .getPermission()); Partition retp3 = client.add_partition(part3); assertNotNull("Unable to create partition " + part3, retp3); - assertEquals(dbPermission, fs.getFileStatus(new Path(retp3.getSd().getLocation())) - .getPermission()); Partition retp4 = client.add_partition(part4); assertNotNull("Unable to create partition " + part4, retp4); - assertEquals(dbPermission, fs.getFileStatus(new Path(retp4.getSd().getLocation())) - .getPermission()); Partition part_get = client.getPartition(dbName, tblName, part.getValues()); if(isThriftClient) { @@ -394,8 +375,6 @@ public abstract class TestHiveMetaStore extends TestCase { // tested retp = client.add_partition(part); assertNotNull("Unable to create partition " + part, retp); - assertEquals(dbPermission, fs.getFileStatus(new Path(retp.getSd().getLocation())) - .getPermission()); // test add_partitions @@ -431,9 +410,8 @@ public abstract class TestHiveMetaStore extends TestCase { // create dir for /mpart5 Path mp5Path = new Path(mpart5.getSd().getLocation()); - warehouse.mkdirs(mp5Path, true); + warehouse.mkdirs(mp5Path); assertTrue(fs.exists(mp5Path)); - assertEquals(dbPermission, fs.getFileStatus(mp5Path).getPermission()); // add_partitions(5,4) : err = duplicate keyvals on mpart4 savedException = null; @@ -2502,7 +2480,7 @@ public abstract class TestHiveMetaStore extends TestCase { //test params //test_param_2 = "50" filter = org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS + - "test_param_2 = \"50\""; + "test_param_2 LIKE \"50\""; tableNames = client.listTableNamesByFilter(dbName, filter, (short)-1); assertEquals(2, tableNames.size()); @@ -2511,30 +2489,31 @@ public abstract class TestHiveMetaStore extends TestCase { //test_param_2 = "75" filter = org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS + - "test_param_2 = \"75\""; + "test_param_2 LIKE \"75\""; tableNames = client.listTableNamesByFilter(dbName, filter, (short)-1); assertEquals(0, tableNames.size()); //key_dne = "50" filter = org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS + - "key_dne = \"50\""; + "key_dne LIKE \"50\""; tableNames = client.listTableNamesByFilter(dbName, filter, (short)-1); assertEquals(0, tableNames.size()); //test_param_1 != "yellow" filter = org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS + - "test_param_1 <> \"yellow\""; + "test_param_1 NOT LIKE \"yellow\""; - tableNames = client.listTableNamesByFilter(dbName, filter, (short) 2); - assertEquals(2, tableNames.size()); + // Commenting as part of HIVE-12274 != and <> are not supported for CLOBs + // tableNames = client.listTableNamesByFilter(dbName, filter, (short) 2); + // assertEquals(2, tableNames.size()); filter = org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS + - "test_param_1 != \"yellow\""; + "test_param_1 NOT LIKE \"yellow\""; - tableNames = client.listTableNamesByFilter(dbName, filter, (short) 2); - assertEquals(2, tableNames.size()); + // tableNames = client.listTableNamesByFilter(dbName, filter, (short) 2); + // assertEquals(2, tableNames.size()); //owner = "testOwner1" and (lastAccessTime = 30 or test_param_1 = "hi") filter = org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_OWNER + @@ -2542,7 +2521,7 @@ public abstract class TestHiveMetaStore extends TestCase { org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_LAST_ACCESS + " = 30 or " + org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS + - "test_param_1 = \"hi\")"; + "test_param_1 LIKE \"hi\")"; tableNames = client.listTableNamesByFilter(dbName, filter, (short)-1); assertEquals(2, tableNames.size()); http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java index a0f18c6..1002be7 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java @@ -128,7 +128,7 @@ public class TestHiveMetaStoreTxns { Assert.assertEquals(ValidTxnList.RangeResponse.NONE, validTxns.isTxnRangeValid(5L, 10L)); - validTxns = new ValidReadTxnList("10:5:4:5:6"); + validTxns = new ValidReadTxnList("10:5:4,5,6:"); Assert.assertEquals(ValidTxnList.RangeResponse.NONE, validTxns.isTxnRangeValid(4,6)); Assert.assertEquals(ValidTxnList.RangeResponse.ALL, @@ -223,15 +223,15 @@ public class TestHiveMetaStoreTxns { @Test public void stringifyValidTxns() throws Exception { // Test with just high water mark - ValidTxnList validTxns = new ValidReadTxnList("1:" + Long.MAX_VALUE + ":"); + ValidTxnList validTxns = new ValidReadTxnList("1:" + Long.MAX_VALUE + "::"); String asString = validTxns.toString(); - Assert.assertEquals("1:" + Long.MAX_VALUE + ":", asString); + Assert.assertEquals("1:" + Long.MAX_VALUE + "::", asString); validTxns = new ValidReadTxnList(asString); Assert.assertEquals(1, validTxns.getHighWatermark()); Assert.assertNotNull(validTxns.getInvalidTransactions()); Assert.assertEquals(0, validTxns.getInvalidTransactions().length); asString = validTxns.toString(); - Assert.assertEquals("1:" + Long.MAX_VALUE + ":", asString); + Assert.assertEquals("1:" + Long.MAX_VALUE + "::", asString); validTxns = new ValidReadTxnList(asString); Assert.assertEquals(1, validTxns.getHighWatermark()); Assert.assertNotNull(validTxns.getInvalidTransactions()); http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java index 6d1673d..7188af6 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java @@ -189,13 +189,13 @@ public class TestMetastoreVersion extends TestCase { // write the given version to metastore private String getVersion(HiveConf conf) throws HiveMetaException { - MetaStoreSchemaInfo schemInfo = new MetaStoreSchemaInfo(metaStoreRoot, conf, "derby"); + MetaStoreSchemaInfo schemInfo = new MetaStoreSchemaInfo(metaStoreRoot, "derby"); return getMetaStoreVersion(); } // write the given version to metastore private void setVersion(HiveConf conf, String version) throws HiveMetaException { - MetaStoreSchemaInfo schemInfo = new MetaStoreSchemaInfo(metaStoreRoot, conf, "derby"); + MetaStoreSchemaInfo schemInfo = new MetaStoreSchemaInfo(metaStoreRoot, "derby"); setMetaStoreVersion(version, "setVersion test"); } http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java index 1ac4d01..3f9eec3 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java @@ -151,15 +151,15 @@ public class TestReplChangeManager { Partition part3 = createPartition(dbName, tblName, columns, values, serdeInfo); client.add_partition(part3); - Path part1Path = new Path(warehouse.getPartitionPath(db, tblName, ImmutableMap.of("dt", "20160101")), "part"); + Path part1Path = new Path(warehouse.getDefaultPartitionPath(db, tblName, ImmutableMap.of("dt", "20160101")), "part"); createFile(part1Path, "p1"); String path1Chksum = ReplChangeManager.getChksumString(part1Path, fs); - Path part2Path = new Path(warehouse.getPartitionPath(db, tblName, ImmutableMap.of("dt", "20160102")), "part"); + Path part2Path = new Path(warehouse.getDefaultPartitionPath(db, tblName, ImmutableMap.of("dt", "20160102")), "part"); createFile(part2Path, "p2"); String path2Chksum = ReplChangeManager.getChksumString(part2Path, fs); - Path part3Path = new Path(warehouse.getPartitionPath(db, tblName, ImmutableMap.of("dt", "20160103")), "part"); + Path part3Path = new Path(warehouse.getDefaultPartitionPath(db, tblName, ImmutableMap.of("dt", "20160103")), "part"); createFile(part3Path, "p3"); String path3Chksum = ReplChangeManager.getChksumString(part3Path, fs); @@ -221,15 +221,15 @@ public class TestReplChangeManager { client.createTable(tbl); - Path filePath1 = new Path(warehouse.getTablePath(db, tblName), "part1"); + Path filePath1 = new Path(warehouse.getDefaultTablePath(db, tblName), "part1"); createFile(filePath1, "f1"); String fileChksum1 = ReplChangeManager.getChksumString(filePath1, fs); - Path filePath2 = new Path(warehouse.getTablePath(db, tblName), "part2"); + Path filePath2 = new Path(warehouse.getDefaultTablePath(db, tblName), "part2"); createFile(filePath2, "f2"); String fileChksum2 = ReplChangeManager.getChksumString(filePath2, fs); - Path filePath3 = new Path(warehouse.getTablePath(db, tblName), "part3"); + Path filePath3 = new Path(warehouse.getDefaultTablePath(db, tblName), "part3"); createFile(filePath3, "f3"); String fileChksum3 = ReplChangeManager.getChksumString(filePath3, fs); http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java new file mode 100644 index 0000000..abf9769 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java @@ -0,0 +1,436 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestAutoPurgeTables { + private static final String driverName = "org.apache.hive.jdbc.HiveDriver"; + private static final String testDbName = "auto_purge_test_db"; + //private static final String testTableName = "auto_purge_test_table"; + private static final String INSERT_OVERWRITE_COMMAND_FORMAT = + "insert overwrite table " + testDbName + ".%s select 1, \"test\""; + private static final String TRUNCATE_TABLE_COMMAND_FORMAT = + "truncate table " + testDbName + ".%s"; + private static final String partitionedColumnName = "partCol"; + private static final String partitionedColumnValue1 = "20090619"; + private static final String INSERT_OVERWRITE_COMMAND_PARTITIONED_FORMAT = + "insert overwrite table " + testDbName + ".%s PARTITION (" + + partitionedColumnName + "=" + partitionedColumnValue1 + ")" + " select 1, \"test\""; + private static final String partitionedColumnValue2 = "20100720"; + private static HiveConf conf; + private static Connection con; + private static MiniHS2 miniHS2; + static final private Logger LOG = LoggerFactory.getLogger("TestAutoPurgeTables"); + + @Rule + public TestName name = new TestName(); + + private static Connection getConnection(String url) throws SQLException { + Connection con1; + con1 = DriverManager.getConnection(url, "", ""); + Assert.assertNotNull("Connection is null", con1); + Assert.assertFalse("Connection should not be closed", con1.isClosed()); + return con1; + } + + private static void createTestTable(Statement stmt, String isAutopurge, boolean isExternal, + boolean isPartitioned, String testTableName) throws SQLException { + String createTablePrefix; + if (isExternal) { + createTablePrefix = "create external table "; + } else { + createTablePrefix = "create table "; + } + if (isPartitioned) { + // create a partitioned table + stmt.execute(createTablePrefix + testDbName + "." + testTableName + " (id int, value string) " + + " partitioned by (" + partitionedColumnName + " STRING)"); + // load data + stmt.execute("insert into " + testDbName + "." + testTableName + " PARTITION (" + + partitionedColumnName + "=" + partitionedColumnValue1 + + ") values (1, \"dummy1\"), (2, \"dummy2\"), (3, \"dummy3\")"); + stmt.execute("insert into " + testDbName + "." + testTableName + " PARTITION (" + + partitionedColumnName + "=" + partitionedColumnValue2 + + ") values (4, \"dummy4\"), (5, \"dummy5\"), (6, \"dummy6\")"); + } else { + // create a table + stmt.execute(createTablePrefix + testDbName + "." + testTableName + " (id int, value string)"); + // load data + stmt.execute("insert into " + testDbName + "." + testTableName + + " values (1, \"dummy1\"), (2, \"dummy2\"), (3, \"dummy3\")"); + } + if (isAutopurge != null) { + stmt.execute("alter table " + testDbName + "." + testTableName + + " set tblproperties (\"auto.purge\"=\"" + isAutopurge + "\")"); + } + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + conf = new HiveConf(TestAutoPurgeTables.class); + // enable trash so it can be tested + conf.setFloat("fs.trash.checkpoint.interval", 30); + conf.setFloat("fs.trash.interval", 30); + // Create test database and base tables once for all the test + miniHS2 = new MiniHS2.Builder().withConf(conf).build(); + miniHS2.start(new HashMap<String, String>()); + Class.forName(driverName); + con = getConnection(miniHS2.getBaseJdbcURL() + ";create=true"); + try (Statement stmt = con.createStatement()) { + Assert.assertNotNull("Statement is null", stmt); + stmt.execute("set hive.support.concurrency = false"); + stmt.execute("drop database if exists " + testDbName + " cascade"); + stmt.execute("create database " + testDbName); + } + } + + @AfterClass + public static void tearDownAfterClass() { + Statement stmt = null; + try { + stmt = con.createStatement(); + // drop test db and its tables and views + stmt.execute("set hive.support.concurrency = false"); + stmt.execute("drop database if exists " + testDbName + " cascade"); + FileSystem fs = FileSystem.get(conf); + fs.deleteOnExit(ShimLoader.getHadoopShims().getCurrentTrashPath(conf, fs)); + } catch (SQLException | IOException e) { + e.printStackTrace(); + } finally { + if (stmt != null) { + try { + stmt.close(); + } catch (SQLException e) { + // + } + } + if (con != null) { + try { + con.close(); + } catch (SQLException e) { + // + } + } + if (miniHS2 != null) { + miniHS2.cleanup(); + miniHS2.stop(); + miniHS2 = null; + } + } + } + + @Before + public void afterTest() throws Exception { + FileSystem fs = FileSystem.get(conf); + Path trashDir = ShimLoader.getHadoopShims().getCurrentTrashPath(conf, fs); + fs.delete(trashDir, true); + } + + /** + * Tests if previous table data skips trash when insert overwrite table .. is run against a table + * which has auto.purge property set + * + * @throws Exception + */ + @Test + public void testAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("true", false, false, false, name.getMethodName()); + } + + /** + * Tests when auto.purge is set to a invalid string, trash should be used for insert overwrite + * queries + * + * @throws Exception + */ + @Test + public void testAutoPurgeInvalid() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("invalid", false, false, false, name.getMethodName()); + } + + /** + * Test when auto.purge property is not set. Data should be moved to trash for insert overwrite + * queries + * + * @throws Exception + */ + @Test + public void testAutoPurgeUnset() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(null, false, false, false, name.getMethodName()); + } + + /** + * Tests if the auto.purge property works correctly for external tables. Old data should skip + * trash when insert overwrite table .. is run when auto.purge is set to true + * + * @throws Exception + */ + @Test + public void testExternalTable() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("true", true, false, false, name.getMethodName()); + } + + /** + * Tests auto.purge when managed table is partitioned. Old data should skip trash when insert + * overwrite table .. is run and auto.purge property is set to true + * + * @throws Exception + */ + @Test + public void testPartitionedTable() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("true", false, true, false, name.getMethodName()); + } + + /** + * Tests auto.purge for an external, partitioned table. Old partition data should skip trash when + * auto.purge is set to true + * + * @throws Exception + */ + @Test + public void testExternalPartitionedTable() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("true", true, true, false, name.getMethodName()); + } + + /** + * Tests when auto.purge is set to false, older data is moved to Trash when insert overwrite table + * .. is run + * + * @throws Exception + */ + @Test + public void testNoAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("false", false, false, false, name.getMethodName()); + } + + /** + * Tests when auto.purge is set to false on a external table, older data is moved to Trash when + * insert overwrite table .. is run + * + * @throws Exception + */ + @Test + public void testExternalNoAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("false", true, false, false, name.getMethodName()); + } + + /** + * Tests when auto.purge is set to false on a partitioned table, older data is moved to Trash when + * insert overwrite table .. is run + * + * @throws Exception + */ + @Test + public void testPartitionedNoAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("false", false, true, false, name.getMethodName()); + } + + /** + * Tests when auto.purge is set to false on a partitioned external table, older data is moved to + * Trash when insert overwrite table .. is run + * + * @throws Exception + */ + @Test + public void testPartitionedExternalNoAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("false", true, true, false, name.getMethodName()); + } + + //truncate on external table is not allowed + @Test(expected = SQLException.class) + public void testTruncatePartitionedExternalNoAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(String.valueOf(false), true, true, true, name.getMethodName()); + } + + //truncate on external table is not allowed + @Test(expected = SQLException.class) + public void testTruncateExternalNoAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(String.valueOf(false), true, false, true, name.getMethodName()); + } + + @Test + public void testTruncatePartitionedNoAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(String.valueOf(false), false, true, true, name.getMethodName()); + } + + @Test + public void testTruncateNoAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(String.valueOf(false), false, false, true, name.getMethodName()); + } + + @Test + public void testTruncateInvalidAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("invalid", false, false, true, name.getMethodName()); + } + + @Test + public void testTruncateUnsetAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(null, false, false, true, name.getMethodName()); + } + + //truncate on external table is not allowed + @Test(expected = SQLException.class) + public void testTruncatePartitionedExternalAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(String.valueOf(true), true, true, true, name.getMethodName()); + } + + //truncate on external table is not allowed + @Test(expected = SQLException.class) + public void testTruncateExternalAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(String.valueOf(true), true, false, true, name.getMethodName()); + } + + @Test + public void testTruncatePartitionedAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(String.valueOf(true), false, true, true, name.getMethodName()); + } + + /** + * Test util method to run the insert overwrite table or truncate table test on a table + * + * @param autoPurgePropValue - string value of the auto.purge property for the test table. Ignored + * if null + * @param isExternal - if set creates a external table for the test + * @param isPartitioned - if set creates a partitioned table for the test + * @param isTruncateTest - if set uses truncate table command for the test. Otherwise uses Insert + * overwrite table command for the test + * @param testTableName - test table name + * @throws Exception + */ + private void testUtil(String autoPurgePropValue, boolean isExternal, boolean isPartitioned, + boolean isTruncateTest, String testTableName) throws Exception { + testUtil(autoPurgePropValue, isExternal, isPartitioned, + !"true".equalsIgnoreCase(autoPurgePropValue), isTruncateTest, testTableName); + } + /** + * Test util method to run the insert overwrite table or truncate table test on a table + * + * @param isAutoPurge - If set, creates a table with auto.purge with the given value + * @param isExternal - if set creates a external table for the test + * @param isPartitioned - if set creates a partitioned table for the test + * @param purgeExpected - if set the assert condition for the test is such that it expectes old + * table data to be moved to trash. If not creates a assert condition to make sure that + * data is not moved to trash + * @param isTruncateTest - if set uses truncate table command for the test. Otherwise uses Insert + * overwrite table command for the test + * @param testTableName - table name for the test table + * @throws Exception + */ + private void testUtil(String isAutoPurge, boolean isExternal, boolean isPartitioned, + boolean purgeExpected, boolean isTruncateTest, String testTableName) throws Exception { + try (Statement stmt = con.createStatement()) { + // create a test table with auto.purge = true + createTestTable(stmt, isAutoPurge, isExternal, isPartitioned, testTableName); + int numFilesInTrashBefore = getTrashFileCount(); + String command = getCommand(isTruncateTest, isPartitioned, testTableName); + stmt.execute(command); + int numFilesInTrashAfter = getTrashFileCount(); + if (purgeExpected) { + Assert.assertTrue( + String.format( + "Data should have been moved to trash. Number of files in trash: before : %d after %d", + numFilesInTrashBefore, numFilesInTrashAfter), + numFilesInTrashBefore < numFilesInTrashAfter); + } else { + Assert.assertEquals( + String.format( + "Data should not have been moved to trash. Number of files in trash: before : %d after %d", + numFilesInTrashBefore, numFilesInTrashAfter), + numFilesInTrashBefore, numFilesInTrashAfter); + } + } + } + + private static String getCommand(boolean isTruncateTest, boolean isPartitioned, String testTableName) { + if (isTruncateTest) { + return String.format(TRUNCATE_TABLE_COMMAND_FORMAT, testTableName); + } else if (isPartitioned) { + return String.format(INSERT_OVERWRITE_COMMAND_PARTITIONED_FORMAT, testTableName); + } else { + return String.format(INSERT_OVERWRITE_COMMAND_FORMAT, testTableName); + } + } + + private int getTrashFileCount() throws Exception { + FileSystem fs = FileSystem.get(conf); + Path trashDir = ShimLoader.getHadoopShims().getCurrentTrashPath(conf, fs); + return getFileCount(fs, trashDir); + } + + private int getFileCount(FileSystem fs, Path path) throws Exception { + try { + int count = 0; + if (!fs.exists(path)) { + return count; + } + RemoteIterator<LocatedFileStatus> lfs = fs.listFiles(path, true); + while (lfs.hasNext()) { + LocatedFileStatus lf = lfs.next(); + LOG.info(lf.getPath().toString()); + if (lf.isFile()) { + count++; + } + } + return count; + } catch (IOException e) { + throw new Exception("Exception while list files on " + path, e); + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java index bfb25aa..ce8fe60 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java @@ -52,6 +52,7 @@ public class TestDDLWithRemoteMetastoreSecondNamenode extends TestCase { private static final String Table4Name = "table4_nondefault_nn"; private static final String Table5Name = "table5_nondefault_nn"; private static final String Table6Name = "table6_nondefault_nn"; + private static final String Table7Name = "table7_nondefault_nn"; private static final String Index1Name = "index1_table1_nondefault_nn"; private static final String Index2Name = "index2_table1_nondefault_nn"; private static final String tmpdir = System.getProperty("test.tmp.dir"); @@ -197,6 +198,27 @@ public class TestDDLWithRemoteMetastoreSecondNamenode extends TestCase { } } + private void alterPartitionAndCheck(Table table, String column, + String value, String location) throws CommandNeedRetryException, HiveException { + assertNotNull(location); + executeQuery("ALTER TABLE " + table.getTableName() + + " PARTITION (" + column + "='" + value + "')" + + " SET LOCATION '" + location + "'"); + HashMap<String, String> partitions = new HashMap<String, String>(); + partitions.put(column, value); + Partition partition = db.getPartition(table, partitions, false); + assertNotNull("Partition object is expected for " + table.getTableName() , partition); + String locationActual = partition.getLocation(); + if (new Path(location).toUri().getScheme() != null) { + assertEquals("Partition should be located in the first filesystem", + fs.makeQualified(new Path(location)).toString(), locationActual); + } + else { + assertEquals("Partition should be located in the second filesystem", + fs2.makeQualified(new Path(location)).toString(), locationActual); + } + } + private Table createTableAndCheck(String tableName, String tableLocation) throws CommandNeedRetryException, HiveException, URISyntaxException { return createTableAndCheck(null, tableName, tableLocation); @@ -294,6 +316,15 @@ public class TestDDLWithRemoteMetastoreSecondNamenode extends TestCase { createTableAndCheck(table1, Table6Name, null); } + public void testAlterPartitionSetLocationNonDefaultNameNode() throws Exception { + assertTrue("Test suite should have been initialized", isInitialized); + String tableLocation = tmppathFs2 + "/" + "test_set_part_loc"; + Table table = createTableAndCheck(Table7Name, tableLocation); + + addPartitionAndCheck(table, "p", "p1", "/tmp/test/1"); + alterPartitionAndCheck(table, "p", "p1", "/tmp/test/2"); + } + public void testCreateDatabaseWithTableNonDefaultNameNode() throws Exception { assertTrue("Test suite should be initialied", isInitialized ); final String tableLocation = tmppathFs2 + "/" + Table3Name; http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java index 0688846..c17ca10 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java @@ -53,10 +53,11 @@ public class TestLocationQueries extends BaseTestQueries { * @return non-zero if it failed */ @Override - public int checkCliDriverResults(String tname) throws Exception { + public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception { File logFile = new File(logDir, tname + ".out"); int failedCount = 0; + StringBuilder fileNames = new StringBuilder("Files failing the location check:"); FileReader fr = new FileReader(logFile); BufferedReader in = new BufferedReader(fr); try { @@ -69,19 +70,20 @@ public class TestLocationQueries extends BaseTestQueries { File f = new File(m.group(1)); if (!f.getName().equals(locationSubdir)) { failedCount++; + fileNames.append(f.getName()).append("\r\n"); } locationCount++; } } // we always have to find at least one location, otw the test is useless if (locationCount == 0) { - return Integer.MAX_VALUE; + return QTestProcessExecResult.create(Integer.MAX_VALUE, "0 locations tested"); } } finally { in.close(); } - return failedCount; + return QTestProcessExecResult.create(failedCount, fileNames.toString()); } public CheckResults(String outDir, String logDir, MiniClusterType miniMr, http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java new file mode 100644 index 0000000..191d4a3 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java @@ -0,0 +1,319 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.apache.hive.service.cli.HiveSQLException; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * This unit test is for testing HIVE-13884 with more complex queries and + * hive.metastore.limit.partition.request enabled. + * It covers cases when the query predicates can be pushed down and the + * number of partitions can be retrieved via directSQL. + * It also covers cases when the number of partitions cannot be retrieved + * via directSQL, so it falls back to ORM. + */ +public class TestMetaStoreLimitPartitionRequest { + + private static final String DB_NAME = "max_partition_test_db"; + private static final String TABLE_NAME = "max_partition_test_table"; + private static int PARTITION_REQUEST_LIMIT = 4; + private static MiniHS2 miniHS2 = null; + private static HiveConf conf; + private Connection hs2Conn = null; + private Statement stmt; + + @BeforeClass + public static void beforeTest() throws Exception { + Class.forName(MiniHS2.getJdbcDriverName()); + conf = new HiveConf(); + DriverManager.setLoginTimeout(0); + + conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + conf.setIntVar(HiveConf.ConfVars.METASTORE_LIMIT_PARTITION_REQUEST, PARTITION_REQUEST_LIMIT); + conf.setBoolVar(HiveConf.ConfVars.METASTORE_INTEGER_JDO_PUSHDOWN, true); + conf.setBoolVar(HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL, true); + conf.setBoolVar(HiveConf.ConfVars.DYNAMICPARTITIONING, true); + conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); + conf.setBoolVar(HiveConf.ConfVars.HIVE_CBO_ENABLED, false); + + miniHS2 = new MiniHS2.Builder().withConf(conf).build(); + Map<String, String> overlayProps = new HashMap<String, String>(); + miniHS2.start(overlayProps); + createDb(); + } + + private static void createDb() throws Exception { + Connection conn = + DriverManager.getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar"); + Statement stmt2 = conn.createStatement(); + stmt2.execute("DROP DATABASE IF EXISTS " + DB_NAME + " CASCADE"); + stmt2.execute("CREATE DATABASE " + DB_NAME); + stmt2.close(); + conn.close(); + } + + @Before + public void setUp() throws Exception { + hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(DB_NAME), + System.getProperty("user.name"), "bar"); + stmt = hs2Conn.createStatement(); + stmt.execute("USE " + DB_NAME); + createTable(); + } + + private void createTable() throws Exception { + String tmpTableName = TABLE_NAME + "_tmp"; + stmt.execute("CREATE TABLE " + tmpTableName + + " (id string, value string, num string, ds date) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n' STORED AS TEXTFILE"); + stmt.execute("INSERT OVERWRITE TABLE " + tmpTableName + + " VALUES ('1', 'value1', '25', '2008-04-09'), ('2', 'value2', '30', '2008-04-09'), " + + "('3', 'value3', '35', '2008-04-09'), ('4', 'value4', '40', '2008-04-09'), " + + "('5', 'value5', '25', '2008-05-09'), ('6', 'value6', '30', '2008-05-09'), " + + "('7', 'value7', '35', '2008-05-09'), ('8', 'value8', '40', '2008-05-09'), " + + "('9', 'value9', '25', '2009-04-09'), ('10', 'value10', '30', '2009-04-09'), " + + "('11', 'value11', '35', '2009-04-09'), ('12', 'value12', '40', '2009-04-09')"); + + stmt.execute("CREATE TABLE " + TABLE_NAME + " (id string, value string) PARTITIONED BY (num string, ds date)"); + stmt.execute("INSERT OVERWRITE TABLE " + TABLE_NAME + " PARTITION (num, ds) SELECT id, value, num, ds FROM " + tmpTableName); + } + + @After + public void tearDown() throws Exception { + String tmpTableName = TABLE_NAME + "_tmp"; + stmt.execute("DROP TABLE IF EXISTS " + TABLE_NAME); + stmt.execute("DROP TABLE IF EXISTS " + tmpTableName); + stmt.execute("DROP TABLE IF EXISTS " + TABLE_NAME + "_num_tmp"); + + if (hs2Conn != null) { + hs2Conn.close(); + } + } + + @AfterClass + public static void afterTest() throws Exception { + if (miniHS2 != null && miniHS2.isStarted()) { + miniHS2.stop(); + } + } + + /* Tests with queries which can be pushed down and executed with directSQL */ + + @Test + public void testSimpleQueryWithDirectSql() throws Exception { + String queryString = "select value from %s where num='25' and ds='2008-04-09'"; + executeQuery(queryString, "value1"); + } + + @Test + public void testMoreComplexQueryWithDirectSql() throws Exception { + String queryString = "select value from %s where (ds between '2009-01-01' and '2009-12-31' and num='25') or (ds between '2008-01-01' and '2008-12-31' and num='30')"; + executeQuery(queryString, "value2", "value6", "value9"); + } + + /* + * Tests with queries which can be pushed down and executed with directSQL, but the number of + * partitions which should be fetched is bigger than the maximum set by the + * hive.metastore.limit.partition.request parameter. + */ + + @Test + public void testSimpleQueryWithDirectSqlTooManyPartitions() throws Exception { + String queryString = "select value from %s where ds>'2008-04-20'"; + executeQueryExceedPartitionLimit(queryString, 8); + } + + @Test + public void testMoreComplexQueryWithDirectSqlTooManyPartitions() throws Exception { + String queryString = "select value from %s where num='25' or (num='30' and ds between '2008-01-01' and '2008-12-31')"; + executeQueryExceedPartitionLimit(queryString, 5); + } + + /* + * Tests with queries which cannot be executed with directSQL, because of type mismatch. The type + * of the num column is string, but the parameters used in the where clause are numbers. After + * falling back to ORM, the number of partitions can be fetched by the + * ObjectStore.getNumPartitionsViaOrmFilter method. + */ + + @Test + public void testQueryWithFallbackToORM1() throws Exception { + String queryString = "select value from %s where num!=25 and num!=35 and num!=40"; + executeQuery(queryString, "value2", "value6", "value10"); + } + + @Test + public void testQueryWithFallbackToORMTooManyPartitions1() throws Exception { + String queryString = "select value from %s where num=30 or num=25"; + executeQueryExceedPartitionLimit(queryString, 6); + } + + /* + * Tests with queries which cannot be executed with directSQL, because of type mismatch. The type + * of the num column is string, but the parameters used in the where clause are numbers. After + * falling back to ORM the number of partitions cannot be fetched by the + * ObjectStore.getNumPartitionsViaOrmFilter method. They are fetched by the + * ObjectStore.getPartitionNamesPrunedByExprNoTxn method. + */ + + @Test + public void testQueryWithFallbackToORM2() throws Exception { + String queryString = "select value from %s where num!=25 and ds='2008-04-09'"; + executeQuery(queryString, "value2", "value3", "value4"); + } + + @Test + public void testQueryWithFallbackToORM3() throws Exception { + String queryString = "select value from %s where num between 26 and 31"; + executeQuery(queryString, "value2", "value6", "value10"); + } + + @Test + public void testQueryWithFallbackToORMTooManyPartitions2() throws Exception { + String queryString = "select value from %s where num!=25 and (ds='2008-04-09' or ds='2008-05-09')"; + executeQueryExceedPartitionLimit(queryString, 6); + } + + @Test + public void testQueryWithFallbackToORMTooManyPartitions3() throws Exception { + String queryString = "select value from %s where num>=30"; + executeQueryExceedPartitionLimit(queryString, 9); + } + + @Test + public void testQueryWithFallbackToORMTooManyPartitions4() throws Exception { + String queryString = "select value from %s where num between 20 and 50"; + executeQueryExceedPartitionLimit(queryString, 12); + } + + /* + * Tests with queries which cannot be executed with directSQL, because the contain like or in. + * After falling back to ORM the number of partitions cannot be fetched by the + * ObjectStore.getNumPartitionsViaOrmFilter method. They are fetched by the + * ObjectStore.getPartitionNamesPrunedByExprNoTxn method. + */ + + @Test + public void testQueryWithInWithFallbackToORM() throws Exception { + setupNumTmpTable(); + String queryString = "select value from %s a where ds='2008-04-09' and a.num in (select value from " + TABLE_NAME + "_num_tmp)"; + executeQuery(queryString, "value1", "value2"); + } + + @Test + public void testQueryWithInWithFallbackToORMTooManyPartitions() throws Exception { + setupNumTmpTable(); + String queryString = "select value from %s a where a.num in (select value from " + TABLE_NAME + "_num_tmp)"; + executeQueryExceedPartitionLimit(queryString, 12); + } + + @Test + public void testQueryWithInWithFallbackToORMTooManyPartitions2() throws Exception { + setupNumTmpTable(); + String queryString = "select value from %s a where a.num in (select value from " + TABLE_NAME + "_num_tmp where value='25')"; + executeQueryExceedPartitionLimit(queryString, 12); + } + + @Test + public void testQueryWithLikeWithFallbackToORMTooManyPartitions() throws Exception { + String queryString = "select value from %s where num like '3%%'"; + executeQueryExceedPartitionLimit(queryString, 6); + } + + private void setupNumTmpTable() throws SQLException { + stmt.execute("CREATE TABLE " + TABLE_NAME + "_num_tmp (value string)"); + stmt.execute("INSERT INTO " + TABLE_NAME + "_num_tmp VALUES ('25')"); + stmt.execute("INSERT INTO " + TABLE_NAME + "_num_tmp VALUES ('30')"); + } + + private void executeQuery(String query, String... expectedValues) throws SQLException { + String queryStr = String.format(query, TABLE_NAME); + ResultSet result = stmt.executeQuery(queryStr); + assertTrue(result != null); + Set<String> expectedValueSet = new HashSet<>(Arrays.asList(expectedValues)); + Set<String> resultValues = getResultValues(result); + String errorMsg = getWrongResultErrorMsg(queryStr, expectedValueSet.toString(), resultValues.toString()); + assertTrue(errorMsg, resultValues.equals(expectedValueSet)); + } + + private Set<String> getResultValues(ResultSet result) throws SQLException { + Set<String> resultValues = new HashSet<>(); + while(result.next()) { + resultValues.add(result.getString(1)); + } + return resultValues; + } + + private void executeQueryExceedPartitionLimit(String query, int expectedPartitionNumber) throws Exception { + try { + String queryStr = String.format(query, TABLE_NAME); + stmt.executeQuery(queryStr); + fail("The query should have failed, because the number of requested partitions are bigger than " + + PARTITION_REQUEST_LIMIT); + } catch (HiveSQLException e) { + String exceedLimitMsg = String.format(HiveMetaStore.PARTITION_NUMBER_EXCEED_LIMIT_MSG, expectedPartitionNumber, + TABLE_NAME, PARTITION_REQUEST_LIMIT, ConfVars.METASTORE_LIMIT_PARTITION_REQUEST.varname); + assertTrue(getWrongExceptionMessage(exceedLimitMsg, e.getMessage()), + e.getMessage().contains(exceedLimitMsg.toString())); + } + } + + private String getWrongResultErrorMsg(String query, String expectedValues, String resultValues) { + StringBuilder errorMsg = new StringBuilder(); + errorMsg.append("The query '"); + errorMsg.append(query); + errorMsg.append("' returned wrong values. It returned the values "); + errorMsg.append(resultValues); + errorMsg.append(" instead of the expected "); + errorMsg.append(expectedValues); + return errorMsg.toString(); + } + + private String getWrongExceptionMessage(String exceedLimitMsg, String exceptionMessage) { + StringBuilder errorMsg = new StringBuilder(); + errorMsg.append("The message of the exception doesn't contain the expected '"); + errorMsg.append(exceedLimitMsg.toString()); + errorMsg.append("'. It is: "); + errorMsg.append(exceptionMessage); + return errorMsg.toString(); + } + +}
