http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java index 0a53259..d227275 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java @@ -17,12 +17,12 @@ */ package org.apache.hive.jdbc; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; -import java.net.URLEncoder; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; @@ -35,7 +35,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.Shell; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; + import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.hadoop.hive.jdbc.SSLTestUtils; + import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -44,10 +47,13 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; + public class TestSSL { + private static final Logger LOG = LoggerFactory.getLogger(TestSSL.class); private static final String LOCALHOST_KEY_STORE_NAME = "keystore.jks"; private static final String EXAMPLEDOTCOM_KEY_STORE_NAME = "keystore_exampledotcom.jks"; @@ -55,19 +61,12 @@ public class TestSSL { private static final String KEY_STORE_TRUST_STORE_PASSWORD = "HiveJdbc"; private static final String JAVA_TRUST_STORE_PROP = "javax.net.ssl.trustStore"; private static final String JAVA_TRUST_STORE_PASS_PROP = "javax.net.ssl.trustStorePassword"; - private static final String HS2_BINARY_MODE = "binary"; - private static final String HS2_HTTP_MODE = "http"; - private static final String HS2_HTTP_ENDPOINT = "cliservice"; - private static final String HS2_BINARY_AUTH_MODE = "NONE"; private MiniHS2 miniHS2 = null; private static HiveConf conf = new HiveConf(); private Connection hs2Conn = null; - private String dataFileDir = conf.get("test.data.files"); + private String dataFileDir = SSLTestUtils.getDataFileDir(); private Map<String, String> confOverlay; - private final String SSL_CONN_PARAMS = "ssl=true;sslTrustStore=" - + URLEncoder.encode(dataFileDir + File.separator + TRUST_STORE_NAME) + ";trustStorePassword=" - + KEY_STORE_TRUST_STORE_PASSWORD; @BeforeClass public static void beforeTest() throws Exception { @@ -83,10 +82,6 @@ public class TestSSL { @Before public void setUp() throws Exception { DriverManager.setLoginTimeout(0); - if (!System.getProperty("test.data.files", "").isEmpty()) { - dataFileDir = System.getProperty("test.data.files"); - } - dataFileDir = dataFileDir.replace('\\', '/').replace("c:", ""); miniHS2 = new MiniHS2.Builder().withConf(conf).cleanupLocalDirOnStartup(false).build(); confOverlay = new HashMap<String, String>(); } @@ -127,9 +122,9 @@ public class TestSSL { // we depend on linux openssl exit codes Assume.assumeTrue(System.getProperty("os.name").toLowerCase().contains("linux")); - setSslConfOverlay(confOverlay); + SSLTestUtils.setSslConfOverlay(confOverlay); // Test in binary mode - setBinaryConfOverlay(confOverlay); + SSLTestUtils.setBinaryConfOverlay(confOverlay); // Start HS2 with SSL miniHS2.start(confOverlay); @@ -146,7 +141,7 @@ public class TestSSL { miniHS2.stop(); // Test in http mode - setHttpConfOverlay(confOverlay); + SSLTestUtils.setHttpConfOverlay(confOverlay); miniHS2.start(confOverlay); // make SSL connection try { @@ -175,13 +170,13 @@ public class TestSSL { */ @Test public void testInvalidConfig() throws Exception { - clearSslConfOverlay(confOverlay); + SSLTestUtils.clearSslConfOverlay(confOverlay); // Test in binary mode - setBinaryConfOverlay(confOverlay); + SSLTestUtils.setBinaryConfOverlay(confOverlay); miniHS2.start(confOverlay); DriverManager.setLoginTimeout(4); try { - hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS), + hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS), System.getProperty("user.name"), "bar"); fail("SSL connection should fail with NON-SSL server"); } catch (SQLException e) { @@ -204,10 +199,10 @@ public class TestSSL { // Test in http mode with ssl properties specified in url System.clearProperty(JAVA_TRUST_STORE_PROP); System.clearProperty(JAVA_TRUST_STORE_PASS_PROP); - setHttpConfOverlay(confOverlay); + SSLTestUtils.setHttpConfOverlay(confOverlay); miniHS2.start(confOverlay); try { - hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS), + hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS), System.getProperty("user.name"), "bar"); fail("SSL connection should fail with NON-SSL server"); } catch (SQLException e) { @@ -223,9 +218,9 @@ public class TestSSL { */ @Test public void testConnectionMismatch() throws Exception { - setSslConfOverlay(confOverlay); + SSLTestUtils.setSslConfOverlay(confOverlay); // Test in binary mode - setBinaryConfOverlay(confOverlay); + SSLTestUtils.setBinaryConfOverlay(confOverlay); miniHS2.start(confOverlay); // Start HS2 with SSL try { @@ -247,7 +242,7 @@ public class TestSSL { miniHS2.stop(); // Test in http mode - setHttpConfOverlay(confOverlay); + SSLTestUtils.setHttpConfOverlay(confOverlay); miniHS2.start(confOverlay); try { hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", ";ssl=false"), @@ -266,23 +261,23 @@ public class TestSSL { */ @Test public void testSSLConnectionWithURL() throws Exception { - setSslConfOverlay(confOverlay); + SSLTestUtils.setSslConfOverlay(confOverlay); // Test in binary mode - setBinaryConfOverlay(confOverlay); + SSLTestUtils.setBinaryConfOverlay(confOverlay); // Start HS2 with SSL miniHS2.start(confOverlay); // make SSL connection - hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS), + hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS), System.getProperty("user.name"), "bar"); hs2Conn.close(); miniHS2.stop(); // Test in http mode - setHttpConfOverlay(confOverlay); + SSLTestUtils.setHttpConfOverlay(confOverlay); miniHS2.start(confOverlay); // make SSL connection - hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS), + hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS), System.getProperty("user.name"), "bar"); hs2Conn.close(); } @@ -293,9 +288,9 @@ public class TestSSL { */ @Test public void testSSLConnectionWithProperty() throws Exception { - setSslConfOverlay(confOverlay); + SSLTestUtils.setSslConfOverlay(confOverlay); // Test in binary mode - setBinaryConfOverlay(confOverlay); + SSLTestUtils.setBinaryConfOverlay(confOverlay); // Start HS2 with SSL miniHS2.start(confOverlay); @@ -308,10 +303,10 @@ public class TestSSL { miniHS2.stop(); // Test in http mode - setHttpConfOverlay(confOverlay); + SSLTestUtils.setHttpConfOverlay(confOverlay); miniHS2.start(confOverlay); // make SSL connection - hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS), + hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default",SSLTestUtils.SSL_CONN_PARAMS), System.getProperty("user.name"), "bar"); hs2Conn.close(); } @@ -322,9 +317,9 @@ public class TestSSL { */ @Test public void testSSLFetch() throws Exception { - setSslConfOverlay(confOverlay); + SSLTestUtils.setSslConfOverlay(confOverlay); // Test in binary mode - setBinaryConfOverlay(confOverlay); + SSLTestUtils.setBinaryConfOverlay(confOverlay); // Start HS2 with SSL miniHS2.start(confOverlay); @@ -332,11 +327,11 @@ public class TestSSL { Path dataFilePath = new Path(dataFileDir, "kv1.txt"); // make SSL connection - hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS), + hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS), System.getProperty("user.name"), "bar"); // Set up test data - setupTestTableWithData(tableName, dataFilePath, hs2Conn); + SSLTestUtils.setupTestTableWithData(tableName, dataFilePath, hs2Conn); Statement stmt = hs2Conn.createStatement(); ResultSet res = stmt.executeQuery("SELECT * FROM " + tableName); @@ -357,20 +352,20 @@ public class TestSSL { */ @Test public void testSSLFetchHttp() throws Exception { - setSslConfOverlay(confOverlay); + SSLTestUtils.setSslConfOverlay(confOverlay); // Test in http mode - setHttpConfOverlay(confOverlay); + SSLTestUtils.setHttpConfOverlay(confOverlay); miniHS2.start(confOverlay); String tableName = "sslTab"; Path dataFilePath = new Path(dataFileDir, "kv1.txt"); // make SSL connection - hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS), + hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS), System.getProperty("user.name"), "bar"); // Set up test data - setupTestTableWithData(tableName, dataFilePath, hs2Conn); + SSLTestUtils.setupTestTableWithData(tableName, dataFilePath, hs2Conn); Statement stmt = hs2Conn.createStatement(); ResultSet res = stmt.executeQuery("SELECT * FROM " + tableName); int rowCount = 0; @@ -393,16 +388,16 @@ public class TestSSL { @Test public void testConnectionWrongCertCN() throws Exception { // This call sets the default ssl params including the correct keystore in the server config - setSslConfOverlay(confOverlay); + SSLTestUtils.setSslConfOverlay(confOverlay); // Replace default keystore with keystore for www.example.com confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname, dataFileDir + File.separator + EXAMPLEDOTCOM_KEY_STORE_NAME); // Binary (TCP) mode - setBinaryConfOverlay(confOverlay); + SSLTestUtils.setBinaryConfOverlay(confOverlay); miniHS2.start(confOverlay); try { hs2Conn = - DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS), + DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS), System.getProperty("user.name"), "bar"); fail("SSL connection, with the server providing wrong certifcate (with CN www.example.com, " + "instead of localhost), should fail"); @@ -415,11 +410,11 @@ public class TestSSL { miniHS2.stop(); // Http mode - setHttpConfOverlay(confOverlay); + SSLTestUtils.setHttpConfOverlay(confOverlay); miniHS2.start(confOverlay); try { hs2Conn = - DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS), + DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS), System.getProperty("user.name"), "bar"); fail("SSL connection, with the server providing wrong certifcate (with CN www.example.com, " + "instead of localhost), should fail"); @@ -439,10 +434,10 @@ public class TestSSL { */ @Test public void testMetastoreWithSSL() throws Exception { - setMetastoreSslConf(conf); - setSslConfOverlay(confOverlay); + SSLTestUtils.setMetastoreSslConf(conf); + SSLTestUtils.setSslConfOverlay(confOverlay); // Test in http mode - setHttpConfOverlay(confOverlay); + SSLTestUtils.setHttpConfOverlay(confOverlay); miniHS2 = new MiniHS2.Builder().withRemoteMetastore().withConf(conf).cleanupLocalDirOnStartup(false).build(); miniHS2.start(confOverlay); @@ -450,11 +445,11 @@ public class TestSSL { Path dataFilePath = new Path(dataFileDir, "kv1.txt"); // make SSL connection - hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS), + hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS), System.getProperty("user.name"), "bar"); // Set up test data - setupTestTableWithData(tableName, dataFilePath, hs2Conn); + SSLTestUtils.setupTestTableWithData(tableName, dataFilePath, hs2Conn); Statement stmt = hs2Conn.createStatement(); ResultSet res = stmt.executeQuery("SELECT * FROM " + tableName); int rowCount = 0; @@ -474,7 +469,7 @@ public class TestSSL { */ @Test public void testMetastoreConnectionWrongCertCN() throws Exception { - setMetastoreSslConf(conf); + SSLTestUtils.setMetastoreSslConf(conf); conf.setVar(ConfVars.HIVE_METASTORE_SSL_KEYSTORE_PATH, dataFileDir + File.separator + EXAMPLEDOTCOM_KEY_STORE_NAME); miniHS2 = new MiniHS2.Builder().withRemoteMetastore().withConf(conf).cleanupLocalDirOnStartup(false).build(); @@ -486,55 +481,4 @@ public class TestSSL { miniHS2.stop(); } - - private void setupTestTableWithData(String tableName, Path dataFilePath, - Connection hs2Conn) throws Exception { - Statement stmt = hs2Conn.createStatement(); - stmt.execute("set hive.support.concurrency = false"); - - stmt.execute("drop table if exists " + tableName); - stmt.execute("create table " + tableName - + " (under_col int comment 'the under column', value string)"); - - // load data - stmt.execute("load data local inpath '" - + dataFilePath.toString() + "' into table " + tableName); - stmt.close(); - } - - private void setSslConfOverlay(Map<String, String> confOverlay) { - confOverlay.put(ConfVars.HIVE_SERVER2_USE_SSL.varname, "true"); - confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname, - dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME); - confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname, - KEY_STORE_TRUST_STORE_PASSWORD); - } - - private void setMetastoreSslConf(HiveConf conf) { - conf.setBoolVar(ConfVars.HIVE_METASTORE_USE_SSL, true); - conf.setVar(ConfVars.HIVE_METASTORE_SSL_KEYSTORE_PATH, - dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME); - conf.setVar(ConfVars.HIVE_METASTORE_SSL_KEYSTORE_PASSWORD, - KEY_STORE_TRUST_STORE_PASSWORD); - conf.setVar(ConfVars.HIVE_METASTORE_SSL_TRUSTSTORE_PATH, - dataFileDir + File.separator + TRUST_STORE_NAME); - conf.setVar(ConfVars.HIVE_METASTORE_SSL_TRUSTSTORE_PASSWORD, - KEY_STORE_TRUST_STORE_PASSWORD); - } - - private void clearSslConfOverlay(Map<String, String> confOverlay) { - confOverlay.put(ConfVars.HIVE_SERVER2_USE_SSL.varname, "false"); - } - - private void setHttpConfOverlay(Map<String, String> confOverlay) { - confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_HTTP_MODE); - confOverlay.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, HS2_HTTP_ENDPOINT); - confOverlay.put(ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true"); - } - - private void setBinaryConfOverlay(Map<String, String> confOverlay) { - confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_BINARY_MODE); - confOverlay.put(ConfVars.HIVE_SERVER2_AUTHENTICATION.varname, HS2_BINARY_AUTH_MODE); - confOverlay.put(ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true"); - } }
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java index e344e0f..1a8337f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java @@ -100,22 +100,6 @@ public class TestOperationLoggingLayout { String row = iter.next()[0].toString(); Assert.assertEquals(true, row.matches("^.*(FATAL|ERROR|WARN|INFO|DEBUG|TRACE).*$")); } - - String queryString = "set hive.server2.logging.operation.level=verbose"; - client.executeStatement(sessionHandle, queryString, null); - operationHandle = client.executeStatement(sessionHandle, sqlCntStar, null); - // just check for first few lines, some log lines are multi-line strings which can break format - // checks below - rowSetLog = client.fetchResults(operationHandle, FetchOrientation.FETCH_FIRST, 10, - FetchType.LOG); - iter = rowSetLog.iterator(); - // verbose pattern is "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n" - while (iter.hasNext()) { - String row = iter.next()[0].toString(); - // just check if the log line starts with date - Assert.assertEquals(true, - row.matches("^\\d{2}[/](0[1-9]|1[012])[/](0[1-9]|[12][0-9]|3[01]).*$")); - } } private SessionHandle setupSession() throws Exception { http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/pom.xml ---------------------------------------------------------------------- diff --git a/itests/pom.xml b/itests/pom.xml index e5b54bf..400075b 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -19,7 +19,7 @@ <parent> <groupId>org.apache.hive</groupId> <artifactId>hive</artifactId> - <version>2.2.0-SNAPSHOT</version> + <version>3.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/qtest-accumulo/pom.xml ---------------------------------------------------------------------- diff --git a/itests/qtest-accumulo/pom.xml b/itests/qtest-accumulo/pom.xml index 31cee36..b7ce283 100644 --- a/itests/qtest-accumulo/pom.xml +++ b/itests/qtest-accumulo/pom.xml @@ -20,7 +20,7 @@ <parent> <groupId>org.apache.hive</groupId> <artifactId>hive-it</artifactId> - <version>2.2.0-SNAPSHOT</version> + <version>3.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/qtest-spark/pom.xml ---------------------------------------------------------------------- diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml index f301504..a506f7f 100644 --- a/itests/qtest-spark/pom.xml +++ b/itests/qtest-spark/pom.xml @@ -20,7 +20,7 @@ <parent> <groupId>org.apache.hive</groupId> <artifactId>hive-it</artifactId> - <version>2.2.0-SNAPSHOT</version> + <version>3.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/qtest/pom.xml ---------------------------------------------------------------------- diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml index 1c3b601..02664f3 100644 --- a/itests/qtest/pom.xml +++ b/itests/qtest/pom.xml @@ -20,7 +20,7 @@ <parent> <groupId>org.apache.hive</groupId> <artifactId>hive-it</artifactId> - <version>2.2.0-SNAPSHOT</version> + <version>3.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java ---------------------------------------------------------------------- diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java index 24eeb9d..4dd17c1e 100644 --- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java +++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java @@ -22,15 +22,15 @@ import java.util.List; import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hive.beeline.Parallelized; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestRule; import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; -@RunWith(Parameterized.class) +@RunWith(Parallelized.class) public class TestBeeLineDriver { static CliAdapter adapter = new CliConfigs.BeeLineConfig().getCliAdapter(); http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index dc02fd4..df947bb 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -71,6 +71,7 @@ minillap.shared.query.files=insert_into1.q,\ orc_merge3.q,\ orc_merge4.q,\ orc_merge_diff_fs.q,\ + parallel_colstats.q,\ unionDistinct_1.q,\ union_type_chk.q,\ cte_2.q,\ @@ -352,6 +353,8 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\ vector_reduce_groupby_decimal.q,\ vector_string_concat.q,\ vector_struct_in.q,\ + vector_udf_character_length.q,\ + vector_udf_octet_length.q,\ vector_varchar_4.q,\ vector_varchar_mapjoin1.q,\ vector_varchar_simple.q,\ @@ -405,6 +408,7 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\ minillap.query.files=acid_bucket_pruning.q,\ bucket5.q,\ bucket6.q,\ + dynamic_semijoin_user_level.q,\ except_distinct.q,\ explainuser_2.q,\ empty_dir_in_table.q,\ @@ -499,6 +503,7 @@ minillaplocal.query.files=acid_globallimit.q,\ groupby2.q,\ hybridgrace_hashjoin_1.q,\ hybridgrace_hashjoin_2.q,\ + is_distinct_from.q,\ infer_bucket_sort_bucketed_table.q,\ input16_cc.q,\ insert_dir_distcp.q,\ @@ -510,12 +515,14 @@ minillaplocal.query.files=acid_globallimit.q,\ join_max_hashtable.q,\ join_nulls.q,\ join_nullsafe.q,\ + join_is_not_distinct_from.q,\ leftsemijoin_mr.q,\ limit_join_transpose.q,\ lineage2.q,\ lineage3.q,\ list_bucket_dml_10.q,\ llap_partitioned.q,\ + llap_vector_nohybridgrace.q,\ load_dyn_part5.q,\ lvj_mapjoin.q,\ mapjoin_decimal.q,\ @@ -567,6 +574,7 @@ minillaplocal.query.files=acid_globallimit.q,\ schema_evol_text_vecrow_table.q,\ selectDistinctStar.q,\ semijoin.q,\ + semijoin_hint.q,\ smb_cache.q,\ special_character_in_tabnames_1.q,\ sqlmerge.q,\ @@ -611,11 +619,26 @@ minillaplocal.query.files=acid_globallimit.q,\ vector_auto_smb_mapjoin_14.q,\ vector_decimal_2.q,\ vector_decimal_udf.q,\ + vector_groupby_cube1.q,\ + vector_groupby_grouping_id1.q,\ + vector_groupby_grouping_id2.q,\ + vector_groupby_grouping_id3.q,\ + vector_groupby_grouping_sets1.q,\ + vector_groupby_grouping_sets2.q,\ + vector_groupby_grouping_sets3.q,\ + vector_groupby_grouping_sets4.q,\ + vector_groupby_grouping_sets5.q,\ + vector_groupby_grouping_sets6.q,\ + vector_groupby_grouping_sets_grouping.q,\ + vector_groupby_grouping_sets_limit.q,\ + vector_groupby_grouping_window.q,\ + vector_groupby_rollup1.q,\ vector_join30.q,\ vector_join_filters.q,\ vector_leftsemi_mapjoin.q,\ vector_number_compare_projection.q,\ vector_partitioned_date_time.q,\ + vector_ptf_part_simple.q,\ vector_udf1.q,\ vectorization_short_regress.q,\ vectorized_dynamic_partition_pruning.q,\ @@ -725,10 +748,22 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\ encryption_drop_view.q \ encryption_drop_partition.q \ encryption_with_trash.q \ - encryption_ctas.q + encryption_ctas.q \ + encryption_auto_purge_tables.q \ + encryption_drop_table_in_encrypted_db.q beeline.positive.include=drop_with_concurrency.q,\ - escape_comments.q + escape_comments.q,\ + smb_mapjoin_1.q,\ + smb_mapjoin_10.q,\ + smb_mapjoin_11.q,\ + smb_mapjoin_12.q,\ + smb_mapjoin_13.q,\ + smb_mapjoin_16.q,\ + smb_mapjoin_2.q,\ + smb_mapjoin_3.q,\ + smb_mapjoin_7.q,\ + select_dummy_source.q minimr.query.negative.files=cluster_tasklog_retrieval.q,\ file_with_header_footer_negative.q,\ @@ -1336,6 +1371,7 @@ spark.query.files=add_part_multiple.q, \ spark.only.query.files=spark_combine_equivalent_work.q,\ spark_dynamic_partition_pruning.q,\ spark_dynamic_partition_pruning_2.q,\ + spark_explainuser_1.q,\ spark_vectorized_dynamic_partition_pruning.q,\ spark_use_file_size_for_mapjoin.q,\ spark_use_op_stats.q @@ -1410,4 +1446,6 @@ spark.query.negative.files=groupby2_map_skew_multi_distinct.q,\ groupby2_multi_distinct.q,\ groupby3_map_skew_multi_distinct.q,\ groupby3_multi_distinct.q,\ - groupby_grouping_sets7.q + groupby_grouping_sets7.q,\ + spark_job_max_tasks.q + http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/test-serde/pom.xml ---------------------------------------------------------------------- diff --git a/itests/test-serde/pom.xml b/itests/test-serde/pom.xml index df0ce8a..bf5f5d2 100644 --- a/itests/test-serde/pom.xml +++ b/itests/test-serde/pom.xml @@ -19,7 +19,7 @@ <parent> <groupId>org.apache.hive</groupId> <artifactId>hive-it</artifactId> - <version>2.2.0-SNAPSHOT</version> + <version>3.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/pom.xml ---------------------------------------------------------------------- diff --git a/itests/util/pom.xml b/itests/util/pom.xml index 3740af8..a44a55c 100644 --- a/itests/util/pom.xml +++ b/itests/util/pom.xml @@ -19,7 +19,7 @@ <parent> <groupId>org.apache.hive</groupId> <artifactId>hive-it</artifactId> - <version>2.2.0-SNAPSHOT</version> + <version>3.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java index 9c97c31..02abe53 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveVariableSource; import org.apache.hadoop.hive.conf.VariableSubstitution; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; @@ -139,12 +140,14 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter { if ((ecode == 0) ^ expectSuccess) { qt.failed(ecode, fname, debugHint); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { qt.failed(e, fname, debugHint); } http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java index 67064b8..1457db0 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java @@ -287,6 +287,7 @@ public class CliConfigs { excludesFrom(testConfigProps, "minimr.query.negative.files"); excludeQuery("authorization_uri_import.q"); + excludeQuery("spark_job_max_tasks.q"); setResultsDir("ql/src/test/results/clientnegative"); setLogDir("itests/qtest/target/qfile-results/clientnegative"); http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java index 3e4b373..73e5632 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java @@ -18,8 +18,10 @@ package org.apache.hadoop.hive.cli.control; import static org.junit.Assert.assertTrue; + import org.apache.hadoop.hive.accumulo.AccumuloQTestUtil; import org.apache.hadoop.hive.accumulo.AccumuloTestSetup; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; import org.junit.AfterClass; @@ -92,13 +94,13 @@ public class CoreAccumuloCliDriver extends CliAdapter { qt.failed(ecode, fname, null); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, null); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput()); } qt.clearPostTestEffects(); - } catch (Throwable e) { + } catch (Exception e) { qt.failed(e, fname, null); } http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java index acc02eb..8c7057c 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java @@ -17,21 +17,23 @@ */ package org.apache.hadoop.hive.cli.control; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.common.base.Strings; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.hooks.PreExecutePrinter; -import org.apache.hive.beeline.qfile.QFile; -import org.apache.hive.beeline.qfile.QFile.QFileBuilder; -import org.apache.hive.beeline.qfile.QFileBeeLineClient; -import org.apache.hive.beeline.qfile.QFileBeeLineClient.QFileClientBuilder; +import org.apache.hive.beeline.QFile; +import org.apache.hive.beeline.QFile.QFileBuilder; +import org.apache.hive.beeline.QFileBeeLineClient; +import org.apache.hive.beeline.QFileBeeLineClient.QFileClientBuilder; import org.apache.hive.jdbc.miniHS2.MiniHS2; import org.junit.AfterClass; import org.junit.BeforeClass; import java.io.File; import java.io.IOException; +import java.sql.SQLException; import java.util.HashMap; public class CoreBeeLineDriver extends CliAdapter { @@ -44,6 +46,7 @@ public class CoreBeeLineDriver extends CliAdapter { private final File testDataDirectory; private final File testScriptDirectory; private boolean overwrite = false; + private boolean rewriteSourceTables = true; private MiniHS2 miniHS2; private QFileClientBuilder clientBuilder; private QFileBuilder fileBuilder; @@ -68,6 +71,10 @@ public class CoreBeeLineDriver extends CliAdapter { if (testOutputOverwrite != null && "true".equalsIgnoreCase(testOutputOverwrite)) { overwrite = true; } + String testRewriteSourceTables = System.getProperty("test.rewrite.source.tables"); + if (testRewriteSourceTables != null && "false".equalsIgnoreCase(testRewriteSourceTables)) { + rewriteSourceTables = false; + } HiveConf hiveConf = new HiveConf(); // We do not need Zookeeper at the moment @@ -92,19 +99,17 @@ public class CoreBeeLineDriver extends CliAdapter { .setPassword("password"); fileBuilder = new QFileBuilder() - .setHiveRootDirectory(hiveRootDirectory) .setLogDirectory(logDirectory) .setQueryDirectory(queryDirectory) .setResultsDirectory(resultsDirectory) - .setScratchDirectoryString(hiveConf.getVar(HiveConf.ConfVars.SCRATCHDIR)) - .setWarehouseDirectoryString(hiveConf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE)); + .setRewriteSourceTables(rewriteSourceTables); runInfraScript(initScript, new File(logDirectory, "init.beeline"), new File(logDirectory, "init.raw")); } protected void runInfraScript(File script, File beeLineOutput, File log) - throws IOException { + throws IOException, SQLException { try (QFileBeeLineClient beeLineClient = clientBuilder.getClient(beeLineOutput)) { beeLineClient.execute( new String[]{ @@ -114,6 +119,9 @@ public class CoreBeeLineDriver extends CliAdapter { "!run " + script, }, log); + } catch (Exception e) { + throw new SQLException("Error running infra script: " + script + + "\nCheck the following logs for details:\n - " + beeLineOutput + "\n - " + log, e); } } @@ -134,28 +142,41 @@ public class CoreBeeLineDriver extends CliAdapter { try (QFileBeeLineClient beeLineClient = clientBuilder.getClient(qFile.getLogFile())) { long startTime = System.currentTimeMillis(); System.err.println(">>> STARTED " + qFile.getName()); - assertTrue("QFile execution failed, see logs for details", beeLineClient.execute(qFile)); - long endTime = System.currentTimeMillis(); - System.err.println(">>> EXECUTED " + qFile.getName() + ":" + (endTime - startTime) / 1000 - + "s"); + beeLineClient.execute(qFile); + + long queryEndTime = System.currentTimeMillis(); + System.err.println(">>> EXECUTED " + qFile.getName() + ": " + (queryEndTime - startTime) + + "ms"); qFile.filterOutput(); long filterEndTime = System.currentTimeMillis(); - System.err.println(">>> FILTERED " + qFile.getName() + ":" + (filterEndTime - endTime) / 1000 - + "s"); + System.err.println(">>> FILTERED " + qFile.getName() + ": " + (filterEndTime - queryEndTime) + + "ms"); if (!overwrite) { - if (qFile.compareResults()) { + QTestProcessExecResult result = qFile.compareResults(); + + long compareEndTime = System.currentTimeMillis(); + System.err.println(">>> COMPARED " + qFile.getName() + ": " + + (compareEndTime - filterEndTime) + "ms"); + if (result.getReturnCode() == 0) { System.err.println(">>> PASSED " + qFile.getName()); } else { System.err.println(">>> FAILED " + qFile.getName()); - fail("Failed diff"); + String messageText = "Client result comparison failed with error code = " + + result.getReturnCode() + " while executing fname=" + qFile.getName() + "\n"; + String messageBody = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + qFile.getDebugHint() : result.getCapturedOutput(); + fail(messageText + messageBody); } } else { qFile.overwriteResults(); System.err.println(">>> PASSED " + qFile.getName()); } + } catch (Exception e) { + throw new Exception("Exception running or analyzing the results of the query file: " + qFile + + "\n" + qFile.getDebugHint(), e); } } http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java index a735346..d59b650 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java @@ -23,7 +23,9 @@ import static org.junit.Assert.fail; import java.util.concurrent.TimeUnit; import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hadoop.hive.util.ElapsedTimeLoggingWrapper; @@ -175,13 +177,15 @@ public class CoreCliDriver extends CliAdapter { failed = true; qt.failed(ecode, fname, debugHint); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { failed = true; - qt.failedDiff(ecode, fname, debugHint); + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { failed = true; qt.failed(e, fname, debugHint); } finally { http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java index 71a02bc..bff81dd 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java @@ -25,6 +25,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import com.google.common.base.Strings; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; @@ -143,12 +145,14 @@ public class CoreCompareCliDriver extends CliAdapter{ } } - ecode = qt.checkCompareCliDriverResults(fname, outputs); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkCompareCliDriverResults(fname, outputs); + if (result.getReturnCode() != 0) { + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { qt.failed(e, fname, debugHint); } http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java index 956a42d..aa0b071 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java @@ -23,6 +23,7 @@ import static org.junit.Assert.fail; import org.apache.hadoop.hive.hbase.HBaseQTestUtil; import org.apache.hadoop.hive.hbase.HBaseTestSetup; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; import org.junit.AfterClass; @@ -120,12 +121,12 @@ public class CoreHBaseCliDriver extends CliAdapter { qt.failed(ecode, fname, null); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, null); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput()); } - } catch (Throwable e) { + } catch (Exception e) { qt.failed(e, fname, null); } http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java index 6225180..8320a80 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java @@ -23,6 +23,7 @@ import static org.junit.Assert.fail; import org.apache.hadoop.hive.hbase.HBaseQTestUtil; import org.apache.hadoop.hive.hbase.HBaseTestSetup; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; import org.junit.AfterClass; @@ -101,13 +102,13 @@ public class CoreHBaseNegativeCliDriver extends CliAdapter { qt.failed(fname, null); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, null); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput()); } qt.clearPostTestEffects(); - } catch (Throwable e) { + } catch (Exception e) { qt.failed(e, fname, null); } http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java index 65b2ce7..438a61e 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java @@ -20,6 +20,8 @@ package org.apache.hadoop.hive.cli.control; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.common.base.Strings; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; @@ -123,12 +125,14 @@ public class CoreNegativeCliDriver extends CliAdapter{ qt.failed(fname, debugHint); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { qt.failed(e, fname, debugHint); } http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java index 8620cde..26b9ce1 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java @@ -22,6 +22,8 @@ package org.apache.hadoop.hive.cli.control; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.common.base.Strings; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; @@ -121,11 +123,13 @@ public class CorePerfCliDriver extends CliAdapter{ if (ecode != 0) { qt.failed(ecode, fname, debugHint); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } - } catch (Throwable e) { + } catch (Exception e) { qt.failed(e, fname, debugHint); } http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java index 6acc7a0..75cce14 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java @@ -18,12 +18,8 @@ package org.apache.hadoop.hive.ql; - -import com.google.common.base.Strings; -import org.apache.commons.lang3.StringUtils; - /** - * Standard output and return code of a process executed during the qtests + * Standard output and return code of a process executed during the qtests. */ public class QTestProcessExecResult { http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index e32ee74..b897ffa 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -39,6 +39,10 @@ import java.io.PrintStream; import java.io.Serializable; import java.io.StringWriter; import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; @@ -51,6 +55,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.Deque; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -59,17 +64,22 @@ import java.util.Properties; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Stream; import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; +import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileStatus; @@ -118,7 +128,6 @@ import org.apache.hadoop.hive.ql.processors.HiveCommand; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hadoop.util.Shell; import org.apache.hive.common.util.StreamPrinter; import org.apache.logging.log4j.util.Strings; import org.apache.tools.ant.BuildException; @@ -167,7 +176,7 @@ public class QTestUtil { private final Set<String> qNoSessionReuseQuerySet; private final Set<String> qJavaVersionSpecificOutput; private static final String SORT_SUFFIX = ".sorted"; - private final HashSet<String> srcTables; + private final Set<String> srcTables; private final Set<String> srcUDFs; private final MiniClusterType clusterType; private final FsType fsType; @@ -200,7 +209,7 @@ public class QTestUtil { } private HBaseTestingUtility utility; - HashSet<String> getSrcTables() { + public static Set<String> getSrcTables() { HashSet<String> srcTables = new HashSet<String>(); // FIXME: moved default value to here...for now // i think this features is never really used from the command line @@ -1395,7 +1404,8 @@ public class QTestUtil { int rc = response.getResponseCode(); if (rc != 0) { - SessionState.get().out.println(response); + SessionState.getConsole().printError(response.toString(), response.getException() != null ? + Throwables.getStackTraceAsString(response.getException()) : ""); } return rc; @@ -1403,7 +1413,7 @@ public class QTestUtil { throw new RuntimeException("Could not get CommandProcessor for command: " + commandName); } } catch (Exception e) { - throw new RuntimeException("Could not execute test command: " + e.getMessage()); + throw new RuntimeException("Could not execute test command", e); } } @@ -1466,10 +1476,6 @@ public class QTestUtil { // Create an instance of hive in order to create the tables testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); db = Hive.get(conf); - // Create dest4 to replace dest4_sequencefile - LinkedList<String> cols = new LinkedList<String>(); - cols.add("key"); - cols.add("value"); // Move all data from dest4_sequencefile to dest4 drv @@ -1480,7 +1486,7 @@ public class QTestUtil { true, true); } - public int checkNegativeResults(String tname, Exception e) throws Exception { + public QTestProcessExecResult checkNegativeResults(String tname, Exception e) throws Exception { String outFileExtension = getOutFileExtension(tname); @@ -1503,16 +1509,17 @@ public class QTestUtil { outfd.write(e.getMessage()); outfd.close(); - int exitVal = executeDiffCommand(outf.getPath(), expf, false, + QTestProcessExecResult result = executeDiffCommand(outf.getPath(), expf, false, qSortSet.contains(qf.getName())); - if (exitVal != 0 && overWrite) { - exitVal = overwriteResults(outf.getPath(), expf); + if (overWrite) { + overwriteResults(outf.getPath(), expf); + return QTestProcessExecResult.createWithoutOutput(0); } - return exitVal; + return result; } - public int checkParseResults(String tname, ASTNode tree) throws Exception { + public QTestProcessExecResult checkParseResults(String tname, ASTNode tree) throws Exception { if (tree != null) { String outFileExtension = getOutFileExtension(tname); @@ -1528,10 +1535,11 @@ public class QTestUtil { outfd.write(tree.toStringTree()); outfd.close(); - int exitVal = executeDiffCommand(outf.getPath(), expf, false, false); + QTestProcessExecResult exitVal = executeDiffCommand(outf.getPath(), expf, false, false); - if (exitVal != 0 && overWrite) { - exitVal = overwriteResults(outf.getPath(), expf); + if (overWrite) { + overwriteResults(outf.getPath(), expf); + return QTestProcessExecResult.createWithoutOutput(0); } return exitVal; @@ -1700,7 +1708,7 @@ public class QTestUtil { }); /* This list may be modified by specific cli drivers to mask strings that change on every test */ - private List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<Pair<Pattern, String>>() {{ + private final List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<Pair<Pattern, String>>() {{ add(toPatternPair("(pblob|s3.?|swift|wasb.?).*hive-staging.*","### BLOBSTORE_STAGING_PATH ###")); }}; @@ -1712,7 +1720,7 @@ public class QTestUtil { patternsWithMaskComments.add(toPatternPair(patternStr, maskComment)); } - public int checkCliDriverResults(String tname) throws Exception { + public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception { assert(qMap.containsKey(tname)); String outFileExtension = getOutFileExtension(tname); @@ -1721,69 +1729,71 @@ public class QTestUtil { File f = new File(logDir, tname + outFileExtension); maskPatterns(planMask, f.getPath()); - int exitVal = executeDiffCommand(f.getPath(), + QTestProcessExecResult exitVal = executeDiffCommand(f.getPath(), outFileName, false, qSortSet.contains(tname)); - if (exitVal != 0 && overWrite) { - exitVal = overwriteResults(f.getPath(), outFileName); + if (overWrite) { + overwriteResults(f.getPath(), outFileName); + return QTestProcessExecResult.createWithoutOutput(0); } return exitVal; } - public int checkCompareCliDriverResults(String tname, List<String> outputs) throws Exception { + public QTestProcessExecResult checkCompareCliDriverResults(String tname, List<String> outputs) + throws Exception { assert outputs.size() > 1; maskPatterns(planMask, outputs.get(0)); for (int i = 1; i < outputs.size(); ++i) { maskPatterns(planMask, outputs.get(i)); - int ecode = executeDiffCommand( + QTestProcessExecResult result = executeDiffCommand( outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname)); - if (ecode != 0) { + if (result.getReturnCode() != 0) { System.out.println("Files don't match: " + outputs.get(i - 1) + " and " + outputs.get(i)); - return ecode; + return result; } } - return 0; + return QTestProcessExecResult.createWithoutOutput(0); } - private static int overwriteResults(String inFileName, String outFileName) throws Exception { + private static void overwriteResults(String inFileName, String outFileName) throws Exception { // This method can be replaced with Files.copy(source, target, REPLACE_EXISTING) // once Hive uses JAVA 7. System.out.println("Overwriting results " + inFileName + " to " + outFileName); - return executeCmd(new String[] { + int result = executeCmd(new String[]{ "cp", getQuotedString(inFileName), getQuotedString(outFileName) - }); + }).getReturnCode(); + if (result != 0) + throw new IllegalStateException("Unexpected error while overwriting " + + inFileName + " with " + outFileName); } - private static int executeDiffCommand(String inFileName, + private static QTestProcessExecResult executeDiffCommand(String inFileName, String outFileName, boolean ignoreWhiteSpace, boolean sortResults ) throws Exception { - int result = 0; + QTestProcessExecResult result; if (sortResults) { // sort will try to open the output file in write mode on windows. We need to // close it first. SessionState ss = SessionState.get(); if (ss != null && ss.out != null && ss.out != System.out) { - ss.out.close(); + ss.out.close(); } String inSorted = inFileName + SORT_SUFFIX; String outSorted = outFileName + SORT_SUFFIX; - result = sortFiles(inFileName, inSorted); - result |= sortFiles(outFileName, outSorted); - if (result != 0) { - System.err.println("ERROR: Could not sort files before comparing"); - return result; - } + sortFiles(inFileName, inSorted); + sortFiles(outFileName, outSorted); + inFileName = inSorted; outFileName = outSorted; } @@ -1813,40 +1823,47 @@ public class QTestUtil { return result; } - private static int sortFiles(String in, String out) throws Exception { - return executeCmd(new String[] { + private static void sortFiles(String in, String out) throws Exception { + int result = executeCmd(new String[]{ "sort", getQuotedString(in), - }, out, null); + }, out, null).getReturnCode(); + if (result != 0) + throw new IllegalStateException("Unexpected error while sorting " + in); } - private static int executeCmd(Collection<String> args) throws Exception { + private static QTestProcessExecResult executeCmd(Collection<String> args) throws Exception { return executeCmd(args, null, null); } - private static int executeCmd(String[] args) throws Exception { + private static QTestProcessExecResult executeCmd(String[] args) throws Exception { return executeCmd(args, null, null); } - private static int executeCmd(Collection<String> args, String outFile, String errFile) throws Exception { + private static QTestProcessExecResult executeCmd(Collection<String> args, String outFile, + String errFile) throws Exception { String[] cmdArray = args.toArray(new String[args.size()]); return executeCmd(cmdArray, outFile, errFile); } - private static int executeCmd(String[] args, String outFile, String errFile) throws Exception { + private static QTestProcessExecResult executeCmd(String[] args, String outFile, + String errFile) throws Exception { System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' ')); PrintStream out = outFile == null ? SessionState.getConsole().getChildOutStream() : - new PrintStream(new FileOutputStream(outFile), true); + new PrintStream(new FileOutputStream(outFile), true, "UTF-8"); PrintStream err = errFile == null ? SessionState.getConsole().getChildErrStream() : - new PrintStream(new FileOutputStream(errFile), true); + new PrintStream(new FileOutputStream(errFile), true, "UTF-8"); Process executor = Runtime.getRuntime().exec(args); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream str = new PrintStream(bos, true, "UTF-8"); + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, err); - StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out); + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out, str); outPrinter.start(); errPrinter.start(); @@ -1864,7 +1881,8 @@ public class QTestUtil { err.close(); } - return result; + return QTestProcessExecResult. + create(result, new String(bos.toByteArray(), StandardCharsets.UTF_8)); } private static String getQuotedString(String str){ @@ -2040,11 +2058,18 @@ public class QTestUtil { qt[i].clearTestSideEffects(); qt[i].cliInit(qfiles[i].getName(), false); qt[i].executeClient(qfiles[i].getName()); - int ecode = qt[i].checkCliDriverResults(qfiles[i].getName()); - if (ecode != 0) { + QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName()); + if (result.getReturnCode() != 0) { failed = true; - System.err.println("Test " + qfiles[i].getName() - + " results check failed with error code " + ecode); + StringBuilder builder = new StringBuilder(); + builder.append("Test ") + .append(qfiles[i].getName()) + .append(" results check failed with error code ") + .append(result.getReturnCode()); + if (Strings.isNotEmpty(result.getCapturedOutput())) { + builder.append(" and diff value ").append(result.getCapturedOutput()); + } + System.err.println(builder.toString()); outputTestFailureHelpMessage(); } qt[i].clearPostTestEffects(); @@ -2091,11 +2116,18 @@ public class QTestUtil { for (int i = 0; i < qfiles.length; i++) { qtThread[i].join(); - int ecode = qt[i].checkCliDriverResults(qfiles[i].getName()); - if (ecode != 0) { + QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName()); + if (result.getReturnCode() != 0) { failed = true; - System.err.println("Test " + qfiles[i].getName() - + " results check failed with error code " + ecode); + StringBuilder builder = new StringBuilder(); + builder.append("Test ") + .append(qfiles[i].getName()) + .append(" results check failed with error code ") + .append(result.getReturnCode()); + if (Strings.isNotEmpty(result.getCapturedOutput())) { + builder.append(" and diff value ").append(result.getCapturedOutput()); + } + System.err.println(builder.toString()); outputTestFailureHelpMessage(); } } @@ -2193,16 +2225,15 @@ public class QTestUtil { public void failedDiff(int ecode, String fname, String debugHint) { String message = - "Client Execution results failed with error code = " + ecode + " while executing fname=" + + "Client Execution succeeded but contained differences " + + "(error code = " + ecode + ") after executing " + fname + (debugHint != null ? (" " + debugHint) : ""); LOG.error(message); Assert.fail(message); } - public void failed(Throwable e, String fname, String debugHint) { + public void failed(Exception e, String fname, String debugHint) { String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null; - System.err.println("Exception: " + e.getMessage()); - e.printStackTrace(); System.err.println("Failed query: " + fname); System.err.flush(); Assert.fail("Unexpected exception " + @@ -2242,9 +2273,6 @@ public class QTestUtil { } br.close(); } catch (Exception e) { - System.err.println("Exception: " + e.getMessage()); - e.printStackTrace(); - System.err.flush(); Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e)); } } @@ -2268,7 +2296,9 @@ public class QTestUtil { String mdbPath = AbstractCliConfig.HIVE_ROOT + "/data/files/tpcds-perf/metastore_export/"; // Setup the table column stats - BufferedReader br = new BufferedReader(new FileReader(new File(AbstractCliConfig.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql"))); + BufferedReader br = new BufferedReader( + new FileReader( + new File(AbstractCliConfig.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql"))); String command; s.execute("DROP TABLE APP.TABLE_PARAMS"); @@ -2294,15 +2324,13 @@ public class QTestUtil { } br.close(); - File tabColStatsCsv = new File(mdbPath+"csv/TAB_COL_STATS.txt"); - File tabParamsCsv = new File(mdbPath+"csv/TABLE_PARAMS.txt"); + java.nio.file.Path tabColStatsCsv = FileSystems.getDefault().getPath(mdbPath, "csv" ,"TAB_COL_STATS.txt.bz2"); + java.nio.file.Path tabParamsCsv = FileSystems.getDefault().getPath(mdbPath, "csv", "TABLE_PARAMS.txt.bz2"); // Set up the foreign key constraints properly in the TAB_COL_STATS data String tmpBaseDir = System.getProperty(TEST_TMP_DIR_PROPERTY); - File tmpFileLoc1 = new File(tmpBaseDir+"/TAB_COL_STATS.txt"); - File tmpFileLoc2 = new File(tmpBaseDir+"/TABLE_PARAMS.txt"); - FileUtils.copyFile(tabColStatsCsv, tmpFileLoc1); - FileUtils.copyFile(tabParamsCsv, tmpFileLoc2); + java.nio.file.Path tmpFileLoc1 = FileSystems.getDefault().getPath(tmpBaseDir, "TAB_COL_STATS.txt"); + java.nio.file.Path tmpFileLoc2 = FileSystems.getDefault().getPath(tmpBaseDir, "TABLE_PARAMS.txt"); class MyComp implements Comparator<String> { @Override @@ -2314,7 +2342,7 @@ public class QTestUtil { } } - SortedMap<String, Integer> tableNameToID = new TreeMap<String, Integer>(new MyComp()); + final SortedMap<String, Integer> tableNameToID = new TreeMap<String, Integer>(new MyComp()); rs = s.executeQuery("SELECT * FROM APP.TBLS"); while(rs.next()) { @@ -2326,29 +2354,73 @@ public class QTestUtil { LOG.debug("Resultset : " + tblName + " | " + tblId); } } - for (Map.Entry<String, Integer> entry : tableNameToID.entrySet()) { - String toReplace1 = ",_" + entry.getKey() + "_" ; - String replacementString1 = ","+entry.getValue(); - String toReplace2 = "_" + entry.getKey() + "_@" ; - String replacementString2 = ""+entry.getValue()+"@"; - try { - String content1 = FileUtils.readFileToString(tmpFileLoc1, "UTF-8"); - content1 = content1.replaceAll(toReplace1, replacementString1); - FileUtils.writeStringToFile(tmpFileLoc1, content1, "UTF-8"); - String content2 = FileUtils.readFileToString(tmpFileLoc2, "UTF-8"); - content2 = content2.replaceAll(toReplace2, replacementString2); - FileUtils.writeStringToFile(tmpFileLoc2, content2, "UTF-8"); - } catch (IOException e) { - LOG.info("Generating file failed", e); + + final Map<String, Map<String, String>> data = new HashMap<>(); + rs = s.executeQuery("select TBLS.TBL_NAME, a.COLUMN_NAME, a.TYPE_NAME from " + + "(select COLUMN_NAME, TYPE_NAME, SDS.SD_ID from APP.COLUMNS_V2 join APP.SDS on SDS.CD_ID = COLUMNS_V2.CD_ID) a" + + " join APP.TBLS on TBLS.SD_ID = a.SD_ID"); + while (rs.next()) { + String tblName = rs.getString(1); + String colName = rs.getString(2); + String typeName = rs.getString(3); + Map<String, String> cols = data.get(tblName); + if (null == cols) { + cols = new HashMap<>(); } + cols.put(colName, typeName); + data.put(tblName, cols); } + BufferedReader reader = new BufferedReader(new InputStreamReader( + new BZip2CompressorInputStream(Files.newInputStream(tabColStatsCsv, StandardOpenOption.READ)))); + + Stream<String> replaced = reader.lines().parallel().map(str-> { + String[] splits = str.split(","); + String tblName = splits[0]; + String colName = splits[1]; + Integer tblID = tableNameToID.get(tblName); + StringBuilder sb = new StringBuilder("default@"+tblName + "@" + colName + "@" + data.get(tblName).get(colName)+"@"); + for (int i = 2; i < splits.length; i++) { + sb.append(splits[i]+"@"); + } + return sb.append(tblID).toString(); + }); + + Files.write(tmpFileLoc1, (Iterable<String>)replaced::iterator); + replaced.close(); + reader.close(); + + BufferedReader reader2 = new BufferedReader(new InputStreamReader( + new BZip2CompressorInputStream(Files.newInputStream(tabParamsCsv, StandardOpenOption.READ)))); + final Map<String,String> colStats = new ConcurrentHashMap<>(); + Stream<String> replacedStream = reader2.lines().parallel().map(str-> { + String[] splits = str.split("_@"); + String tblName = splits[0]; + Integer tblId = tableNameToID.get(tblName); + Map<String,String> cols = data.get(tblName); + StringBuilder sb = new StringBuilder(); + sb.append("{\"COLUMN_STATS\":{"); + for (String colName : cols.keySet()) { + sb.append("\""+colName+"\":\"true\","); + } + sb.append("},\"BASIC_STATS\":\"true\"}"); + colStats.put(tblId.toString(), sb.toString()); + + return tblId.toString() + "@" + splits[1]; + }); + + Files.write(tmpFileLoc2, (Iterable<String>)replacedStream::iterator); + Files.write(tmpFileLoc2, (Iterable<String>)colStats.entrySet().stream() + .map(map->map.getKey()+"@COLUMN_STATS_ACCURATE@"+map.getValue())::iterator, StandardOpenOption.APPEND); + + replacedStream.close(); + reader2.close(); // Load the column stats and table params with 30 TB scale - String importStatement1 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE(null, '" + "TAB_COL_STATS" + - "', '" + tmpFileLoc1.getAbsolutePath() + - "', ',', null, 'UTF-8', 1)"; - String importStatement2 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE(null, '" + "TABLE_PARAMS" + - "', '" + tmpFileLoc2.getAbsolutePath() + + String importStatement1 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TAB_COL_STATS" + + "', '" + tmpFileLoc1.toAbsolutePath().toString() + + "', '@', null, 'UTF-8', 1)"; + String importStatement2 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TABLE_PARAMS" + + "', '" + tmpFileLoc2.toAbsolutePath().toString() + "', '@', null, 'UTF-8', 1)"; try { PreparedStatement psImport1 = conn.prepareStatement(importStatement1); http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java index 8dba0bb..31f69a3 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java @@ -21,9 +21,12 @@ import static org.junit.Assert.fail; import java.io.Serializable; import java.util.List; + +import com.google.common.base.Strings; import org.apache.hadoop.hive.cli.control.AbstractCliConfig; import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hadoop.hive.ql.exec.Task; @@ -106,18 +109,20 @@ public class CoreParseNegative extends CliAdapter{ fail("Unexpected success for query: " + fname + debugHint); } catch (ParseException pe) { - int ecode = qt.checkNegativeResults(fname, pe); - if (ecode != 0) { - qt.failed(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkNegativeResults(fname, pe); + if (result.getReturnCode() != 0) { + qt.failed(result.getReturnCode(), fname, result.getCapturedOutput() + "\r\n" + debugHint); } } catch (SemanticException se) { - int ecode = qt.checkNegativeResults(fname, se); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkNegativeResults(fname, se); + if (result.getReturnCode() != 0) { + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { qt.failed(e, fname, debugHint); } http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java b/itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java new file mode 100644 index 0000000..a819e26 --- /dev/null +++ b/itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline; + +import org.junit.runners.Parameterized; +import org.junit.runners.model.RunnerScheduler; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +/** + * Class to run Parameterized test in parallel. + * Source: http://hwellmann.blogspot.hu/2009/12/running-parameterized-junit-tests-in.html + */ +public class Parallelized extends Parameterized { + private static class ThreadPoolScheduler implements RunnerScheduler { + private ExecutorService executor; + + public ThreadPoolScheduler() { + String threads = System.getProperty("junit.parallel.threads"); + int numThreads = Runtime.getRuntime().availableProcessors(); + if (threads != null) { + numThreads = Integer.parseInt(threads); + } + executor = Executors.newFixedThreadPool(numThreads); + } + + @Override + public void finished() { + executor.shutdown(); + try { + executor.awaitTermination(10, TimeUnit.MINUTES); + } catch (InterruptedException exc) { + throw new RuntimeException(exc); + } + } + + @Override + public void schedule(Runnable childStatement) { + executor.submit(childStatement); + } + } + + public Parallelized(Class klass) throws Throwable { + super(klass); + setScheduler(new ThreadPoolScheduler()); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hive/beeline/QFile.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java new file mode 100644 index 0000000..0bde529 --- /dev/null +++ b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java @@ -0,0 +1,333 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.beeline; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; +import org.apache.hadoop.hive.ql.QTestUtil; +import org.apache.hadoop.util.Shell; +import org.apache.hive.common.util.StreamPrinter; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * Class for representing a Query and the connected files. It provides accessors for the specific + * input and output files, and provides methods for filtering the output of the runs. + */ +public final class QFile { + private static final Set<String> srcTables = QTestUtil.getSrcTables(); + private static final String DEBUG_HINT = + "The following files can help you identifying the problem:%n" + + " - Query file: %1s%n" + + " - Raw output file: %2s%n" + + " - Filtered output file: %3s%n" + + " - Expected output file: %4s%n" + + " - Client log file: %5s%n" + + " - Client log files before the test: %6s%n" + + " - Client log files after the test: %7s%n" + + " - Hiveserver2 log file: %8s%n"; + private static final String USE_COMMAND_WARNING = + "The query file %1s contains \"%2s\" command.%n" + + "The source table name rewrite is turned on, so this might cause problems when the used " + + "database contains tables named any of the following: " + srcTables + "%n" + + "To turn off the table name rewrite use -Dtest.rewrite.source.tables=false%n"; + + private static final Pattern USE_PATTERN = + Pattern.compile("^\\s*use\\s.*", Pattern.CASE_INSENSITIVE); + + private static final String MASK_PATTERN = "#### A masked pattern was here ####\n"; + + private String name; + private File inputFile; + private File rawOutputFile; + private File outputFile; + private File expectedOutputFile; + private File logFile; + private File beforeExecuteLogFile; + private File afterExecuteLogFile; + private static RegexFilterSet staticFilterSet = getStaticFilterSet(); + private RegexFilterSet specificFilterSet; + private boolean rewriteSourceTables; + + private QFile() {} + + public String getName() { + return name; + } + + public File getInputFile() { + return inputFile; + } + + public File getRawOutputFile() { + return rawOutputFile; + } + + public File getOutputFile() { + return outputFile; + } + + public File getExpectedOutputFile() { + return expectedOutputFile; + } + + public File getLogFile() { + return logFile; + } + + public File getBeforeExecuteLogFile() { + return beforeExecuteLogFile; + } + + public File getAfterExecuteLogFile() { + return afterExecuteLogFile; + } + + public String getDebugHint() { + return String.format(DEBUG_HINT, inputFile, rawOutputFile, outputFile, expectedOutputFile, + logFile, beforeExecuteLogFile, afterExecuteLogFile, + "./itests/qtest/target/tmp/log/hive.log"); + } + + /** + * Filters the sql commands if necessary. + * @param commands The array of the sql commands before filtering + * @return The filtered array of the sql command strings + * @throws IOException File read error + */ + public String[] filterCommands(String[] commands) throws IOException { + if (rewriteSourceTables) { + for (int i=0; i<commands.length; i++) { + if (USE_PATTERN.matcher(commands[i]).matches()) { + System.err.println(String.format(USE_COMMAND_WARNING, inputFile, commands[i])); + } + commands[i] = replaceTableNames(commands[i]); + } + } + return commands; + } + + /** + * Replace the default src database TABLE_NAMEs in the queries with default.TABLE_NAME, like + * src->default.src, srcpart->default.srcpart, so the queries could be run even if the used + * database is query specific. This change is only a best effort, since we do not want to parse + * the queries, we could not be sure that we do not replace other strings which are not + * tablenames. Like 'select src from othertable;'. The q files containing these commands should + * be excluded. Only replace the tablenames, if rewriteSourceTables is set. + * @param source The original query string + * @return The query string where the tablenames are replaced + */ + private String replaceTableNames(String source) { + for (String table : srcTables) { + source = source.replaceAll("(?is)(\\s+)" + table + "([\\s;\\n\\)])", "$1default." + table + + "$2"); + } + return source; + } + + /** + * The result contains the original queries. To revert them to the original form remove the + * 'default' from every default.TABLE_NAME, like default.src->src, default.srcpart->srcpart. + * @param source The original query output + * @return The query output where the tablenames are replaced + */ + private String revertReplaceTableNames(String source) { + for (String table : srcTables) { + source = source.replaceAll("(?is)(\\s+)default\\." + table + "([\\s;\\n\\)])", "$1" + table + + "$2"); + } + return source; + } + + public void filterOutput() throws IOException { + String rawOutput = FileUtils.readFileToString(rawOutputFile, "UTF-8"); + if (rewriteSourceTables) { + rawOutput = revertReplaceTableNames(rawOutput); + } + String filteredOutput = staticFilterSet.filter(specificFilterSet.filter(rawOutput)); + FileUtils.writeStringToFile(outputFile, filteredOutput); + } + + public QTestProcessExecResult compareResults() throws IOException, InterruptedException { + if (!expectedOutputFile.exists()) { + throw new IOException("Expected results file does not exist: " + expectedOutputFile); + } + return executeDiff(); + } + + public void overwriteResults() throws IOException { + FileUtils.copyFile(outputFile, expectedOutputFile); + } + + private QTestProcessExecResult executeDiff() throws IOException, InterruptedException { + List<String> diffCommandArgs = new ArrayList<String>(); + diffCommandArgs.add("diff"); + + // Text file comparison + diffCommandArgs.add("-a"); + + if (Shell.WINDOWS) { + // Ignore changes in the amount of white space + diffCommandArgs.add("-b"); + + // Files created on Windows machines have different line endings + // than files created on Unix/Linux. Windows uses carriage return and line feed + // ("\r\n") as a line ending, whereas Unix uses just line feed ("\n"). + // Also StringBuilder.toString(), Stream to String conversions adds extra + // spaces at the end of the line. + diffCommandArgs.add("--strip-trailing-cr"); // Strip trailing carriage return on input + diffCommandArgs.add("-B"); // Ignore changes whose lines are all blank + } + + // Add files to compare to the arguments list + diffCommandArgs.add(getQuotedString(expectedOutputFile)); + diffCommandArgs.add(getQuotedString(outputFile)); + + System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(diffCommandArgs, + ' ')); + Process executor = Runtime.getRuntime().exec(diffCommandArgs.toArray( + new String[diffCommandArgs.size()])); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bos, true, "UTF-8"); + + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, System.err); + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out, out); + + outPrinter.start(); + errPrinter.start(); + + int result = executor.waitFor(); + + outPrinter.join(); + errPrinter.join(); + + executor.waitFor(); + + return QTestProcessExecResult.create(result, new String(bos.toByteArray(), + StandardCharsets.UTF_8)); + } + + private static String getQuotedString(File file) { + return Shell.WINDOWS ? String.format("\"%s\"", file.getAbsolutePath()) : file.getAbsolutePath(); + } + + private static class Filter { + private final Pattern pattern; + private final String replacement; + + public Filter(Pattern pattern, String replacement) { + this.pattern = pattern; + this.replacement = replacement; + } + } + + private static class RegexFilterSet { + private final List<Filter> regexFilters = new ArrayList<Filter>(); + + public RegexFilterSet addFilter(String regex, String replacement) { + regexFilters.add(new Filter(Pattern.compile(regex), replacement)); + return this; + } + + public String filter(String input) { + for (Filter filter : regexFilters) { + input = filter.pattern.matcher(input).replaceAll(filter.replacement); + } + return input; + } + } + + // These are the filters which are common for every QTest. + // Check specificFilterSet for QTest specific ones. + private static RegexFilterSet getStaticFilterSet() { + // Pattern to remove the timestamp and other infrastructural info from the out file + return new RegexFilterSet() + .addFilter("Reading log file: .*\n", "") + .addFilter("INFO : ", "") + .addFilter(".*/tmp/.*\n", MASK_PATTERN) + .addFilter(".*file:.*\n", MASK_PATTERN) + .addFilter(".*file\\..*\n", MASK_PATTERN) + .addFilter(".*CreateTime.*\n", MASK_PATTERN) + .addFilter(".*transient_lastDdlTime.*\n", MASK_PATTERN) + .addFilter("(?s)(" + MASK_PATTERN + ")+", MASK_PATTERN); + } + + /** + * Builder to generate QFile objects. After initializing the builder it is possible the + * generate the next QFile object using it's name only. + */ + public static class QFileBuilder { + private File queryDirectory; + private File logDirectory; + private File resultsDirectory; + private boolean rewriteSourceTables; + + public QFileBuilder() { + } + + public QFileBuilder setQueryDirectory(File queryDirectory) { + this.queryDirectory = queryDirectory; + return this; + } + + public QFileBuilder setLogDirectory(File logDirectory) { + this.logDirectory = logDirectory; + return this; + } + + public QFileBuilder setResultsDirectory(File resultsDirectory) { + this.resultsDirectory = resultsDirectory; + return this; + } + + public QFileBuilder setRewriteSourceTables(boolean rewriteSourceTables) { + this.rewriteSourceTables = rewriteSourceTables; + return this; + } + + public QFile getQFile(String name) throws IOException { + QFile result = new QFile(); + result.name = name; + result.inputFile = new File(queryDirectory, name + ".q"); + result.rawOutputFile = new File(logDirectory, name + ".q.out.raw"); + result.outputFile = new File(logDirectory, name + ".q.out"); + result.expectedOutputFile = new File(resultsDirectory, name + ".q.out"); + result.logFile = new File(logDirectory, name + ".q.beeline"); + result.beforeExecuteLogFile = new File(logDirectory, name + ".q.beforeExecute.log"); + result.afterExecuteLogFile = new File(logDirectory, name + ".q.afterExecute.log"); + result.rewriteSourceTables = rewriteSourceTables; + result.specificFilterSet = new RegexFilterSet() + .addFilter("(PREHOOK|POSTHOOK): (Output|Input): database:" + name + "\n", + "$1: $2: database:default\n") + .addFilter("(PREHOOK|POSTHOOK): (Output|Input): " + name + "@", "$1: $2: default@") + .addFilter("name(:?) " + name + "\\.(.*)\n", "name$1 default.$2\n") + .addFilter("/" + name + ".db/", "/"); + return result; + } + } +}
