HIVE-14782: Improve runtime of NegativeMinimrCliDriver (Prasanth Jayachandran reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/66af7643 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/66af7643 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/66af7643 Branch: refs/heads/hive-14535 Commit: 66af76435b5a9852f3fa1b8ce9a12c01128737fa Parents: 0c392b1 Author: Prasanth Jayachandran <prasan...@apache.org> Authored: Wed Sep 21 12:01:40 2016 -0700 Committer: Prasanth Jayachandran <prasan...@apache.org> Committed: Wed Sep 21 12:01:40 2016 -0700 ---------------------------------------------------------------------- itests/src/test/resources/testconfiguration.properties | 2 -- .../org/apache/hadoop/hive/cli/control/CliConfigs.java | 4 ++-- .../clientnegative/mapreduce_stack_trace_hadoop20.q | 13 ------------- .../mapreduce_stack_trace_turnoff_hadoop20.q | 13 ------------- .../mapreduce_stack_trace_hadoop20.q.out | 13 ------------- .../mapreduce_stack_trace_turnoff_hadoop20.q.out | 5 ----- .../src/test/resources/test-configuration2.properties | 3 +-- 7 files changed, 3 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/66af7643/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 4597ace..e810a58 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -698,9 +698,7 @@ minimr.query.negative.files=cluster_tasklog_retrieval.q,\ file_with_header_footer_negative.q,\ local_mapred_error_cache.q,\ mapreduce_stack_trace.q,\ - mapreduce_stack_trace_hadoop20.q,\ mapreduce_stack_trace_turnoff.q,\ - mapreduce_stack_trace_turnoff_hadoop20.q,\ minimr_broken_pipe.q,\ table_nonprintable_negative.q,\ udf_local_resource.q http://git-wip-us.apache.org/repos/asf/hive/blob/66af7643/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java index 69c4974..d74f51a 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java @@ -276,8 +276,8 @@ public class CliConfigs { setResultsDir("ql/src/test/results/clientnegative"); setLogDir("itests/qtest/target/qfile-results/clientnegative"); - setInitScript("q_test_init.sql"); - setCleanupScript("q_test_cleanup.sql"); + setInitScript("q_test_init_for_encryption.sql"); + setCleanupScript("q_test_cleanup_for_encryption.sql"); setHiveConfDir(""); setClusterType(MiniClusterType.mr); http://git-wip-us.apache.org/repos/asf/hive/blob/66af7643/ql/src/test/queries/clientnegative/mapreduce_stack_trace_hadoop20.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/mapreduce_stack_trace_hadoop20.q b/ql/src/test/queries/clientnegative/mapreduce_stack_trace_hadoop20.q deleted file mode 100644 index 9d0548c..0000000 --- a/ql/src/test/queries/clientnegative/mapreduce_stack_trace_hadoop20.q +++ /dev/null @@ -1,13 +0,0 @@ -set hive.exec.mode.local.auto=false; -set hive.exec.job.debug.capture.stacktraces=true; -set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.VerifySessionStateStackTracesHook; - -FROM src SELECT TRANSFORM(key, value) USING 'script_does_not_exist' AS (key, value); - --- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.23) --- Hadoop 0.23 changes the getTaskDiagnostics behavior --- The Error Code of hive failure MapReduce job changes --- In Hadoop 0.20 --- Hive failure MapReduce job gets 20000 as Error Code --- In Hadoop 0.23 --- Hive failure MapReduce job gets 2 as Error Code http://git-wip-us.apache.org/repos/asf/hive/blob/66af7643/ql/src/test/queries/clientnegative/mapreduce_stack_trace_turnoff_hadoop20.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/mapreduce_stack_trace_turnoff_hadoop20.q b/ql/src/test/queries/clientnegative/mapreduce_stack_trace_turnoff_hadoop20.q deleted file mode 100644 index e319944..0000000 --- a/ql/src/test/queries/clientnegative/mapreduce_stack_trace_turnoff_hadoop20.q +++ /dev/null @@ -1,13 +0,0 @@ -set hive.exec.mode.local.auto=false; -set hive.exec.job.debug.capture.stacktraces=false; -set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.VerifySessionStateStackTracesHook; - -FROM src SELECT TRANSFORM(key, value) USING 'script_does_not_exist' AS (key, value); - --- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.23) --- Hadoop 0.23 changes the getTaskDiagnostics behavior --- The Error Code of hive failure MapReduce job changes --- In Hadoop 0.20 --- Hive failure MapReduce job gets 20000 as Error Code --- In Hadoop 0.23 --- Hive failure MapReduce job gets 2 as Error Code http://git-wip-us.apache.org/repos/asf/hive/blob/66af7643/ql/src/test/results/clientnegative/mapreduce_stack_trace_hadoop20.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/mapreduce_stack_trace_hadoop20.q.out b/ql/src/test/results/clientnegative/mapreduce_stack_trace_hadoop20.q.out deleted file mode 100644 index dda4216..0000000 --- a/ql/src/test/results/clientnegative/mapreduce_stack_trace_hadoop20.q.out +++ /dev/null @@ -1,13 +0,0 @@ -PREHOOK: query: FROM src SELECT TRANSFORM(key, value) USING 'script_does_not_exist' AS (key, value) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -FATAL org.apache.hadoop.hive.ql.exec.mr.ExecMapper: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row {"key":"238","value":"val_238"} -Hive Runtime Error while processing row {"key":"238","value":"val_238"} -FATAL org.apache.hadoop.hive.ql.exec.mr.ExecMapper: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row {"key":"238","value":"val_238"} -Hive Runtime Error while processing row {"key":"238","value":"val_238"} -FATAL org.apache.hadoop.hive.ql.exec.mr.ExecMapper: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row {"key":"238","value":"val_238"} -Hive Runtime Error while processing row {"key":"238","value":"val_238"} -FATAL org.apache.hadoop.hive.ql.exec.mr.ExecMapper: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row {"key":"238","value":"val_238"} -Hive Runtime Error while processing row {"key":"238","value":"val_238"} -FAILED: Execution Error, return code 20000 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask. Unable to initialize custom script. http://git-wip-us.apache.org/repos/asf/hive/blob/66af7643/ql/src/test/results/clientnegative/mapreduce_stack_trace_turnoff_hadoop20.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/mapreduce_stack_trace_turnoff_hadoop20.q.out b/ql/src/test/results/clientnegative/mapreduce_stack_trace_turnoff_hadoop20.q.out deleted file mode 100644 index dfc8f54..0000000 --- a/ql/src/test/results/clientnegative/mapreduce_stack_trace_turnoff_hadoop20.q.out +++ /dev/null @@ -1,5 +0,0 @@ -PREHOOK: query: FROM src SELECT TRANSFORM(key, value) USING 'script_does_not_exist' AS (key, value) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -FAILED: Execution Error, return code 20000 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask. Unable to initialize custom script. http://git-wip-us.apache.org/repos/asf/hive/blob/66af7643/testutils/ptest2/src/test/resources/test-configuration2.properties ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/test/resources/test-configuration2.properties b/testutils/ptest2/src/test/resources/test-configuration2.properties index 19e1ac2..bb4e77c 100644 --- a/testutils/ptest2/src/test/resources/test-configuration2.properties +++ b/testutils/ptest2/src/test/resources/test-configuration2.properties @@ -107,7 +107,6 @@ qFileTest.miniMrNegative.include = normal qFileTest.miniMrNegative.isolate = flaky # normal are tests that run in minimr mode via build-common.xml qFileTest.miniMrNegative.groups.normal = mainProperties.${minimr.query.negative.files} -qFileTest.miniMrNegative.groups.flaky = mapreduce_stack_trace_hadoop20.q qFileTest.clientNegative.driver = TestNegativeCliDriver qFileTest.clientNegative.directory = ql/src/test/queries/clientnegative @@ -118,7 +117,7 @@ qFileTest.clientNegative.exclude = miniMrNormal failing # Disable for HIVE-4941 as this tests runs via ant test #qFileTest.clientNegative.groups.failing = stats_aggregator_error_1.q # normal are run via minimr -qFileTest.clientNegative.groups.miniMrNormal = cluster_tasklog_retrieval.q minimr_broken_pipe.q mapreduce_stack_trace.q mapreduce_stack_trace_turnoff.q mapreduce_stack_trace_hadoop20.q mapreduce_stack_trace_turnoff_hadoop20.q +qFileTest.clientNegative.groups.miniMrNormal = cluster_tasklog_retrieval.q minimr_broken_pipe.q mapreduce_stack_trace.q mapreduce_stack_trace_turnoff.q qFileTest.hbasePositive.driver = TestHBaseCliDriver qFileTest.hbasePositive.directory = hbase-handler/src/test/queries/positive
