IMPALA-5588: Reduce the frequency of fault injection Previously, the fault injection utility will inject a fault on every 3 RPC calls for ReportExecStatus() RPCs. As shown in IMPALA-5588, with an unfortunate sequence in which other RPCs happen between the retry of ReportExecStatus() RPC in QueryState::ReportExecStatusAux(), ReportExecStatus() can hit injected faults 3 times in a row, causing the query to be cancelled in QueryState::ReportExecStatusAux().
This change fixes the problem by reducing the fault injection frequency to once every 16 RPC calls for ReportExecStatus(), CancelQueryFInstances() and ExecQueryFInstances() RPCs. Also incorporated the fix by Michael Brown for a python bug in test_rpc_exception.py so tests hitting unexpected exception will re-throw that exception for better diagnosis on test failure. Change-Id: I0ce4445e8552a22f23371bed1196caf7d0a3f312 Reviewed-on: http://gerrit.cloudera.org:8080/7310 Reviewed-by: Michael Ho <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/fb8ea6a9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/fb8ea6a9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/fb8ea6a9 Branch: refs/heads/master Commit: fb8ea6a9ccaedc41a71f6f6dcb367fc1facd73b6 Parents: 3ca5a76 Author: Michael Ho <[email protected]> Authored: Tue Jun 27 14:07:27 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Fri Jun 30 09:27:47 2017 +0000 ---------------------------------------------------------------------- be/src/runtime/backend-client.h | 10 +++++----- tests/custom_cluster/test_rpc_exception.py | 2 ++ 2 files changed, 7 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fb8ea6a9/be/src/runtime/backend-client.h ---------------------------------------------------------------------- diff --git a/be/src/runtime/backend-client.h b/be/src/runtime/backend-client.h index c2d3adc..136a71e 100644 --- a/be/src/runtime/backend-client.h +++ b/be/src/runtime/backend-client.h @@ -48,7 +48,7 @@ class ImpalaBackendClient : public ImpalaInternalServiceClient { void ExecQueryFInstances(TExecQueryFInstancesResult& _return, const TExecQueryFInstancesParams& params, bool* send_done) { DCHECK(!*send_done); - FAULT_INJECTION_SEND_RPC_EXCEPTION(5); + FAULT_INJECTION_SEND_RPC_EXCEPTION(16); ImpalaInternalServiceClient::send_ExecQueryFInstances(params); *send_done = true; // Cannot inject fault on recv() side as the callers cannot handle it. @@ -58,20 +58,20 @@ class ImpalaBackendClient : public ImpalaInternalServiceClient { void ReportExecStatus(TReportExecStatusResult& _return, const TReportExecStatusParams& params, bool* send_done) { DCHECK(!*send_done); - FAULT_INJECTION_SEND_RPC_EXCEPTION(3); + FAULT_INJECTION_SEND_RPC_EXCEPTION(16); ImpalaInternalServiceClient::send_ReportExecStatus(params); *send_done = true; - FAULT_INJECTION_RECV_RPC_EXCEPTION(3); + FAULT_INJECTION_RECV_RPC_EXCEPTION(16); ImpalaInternalServiceClient::recv_ReportExecStatus(_return); } void CancelQueryFInstances(TCancelQueryFInstancesResult& _return, const TCancelQueryFInstancesParams& params, bool* send_done) { DCHECK(!*send_done); - FAULT_INJECTION_SEND_RPC_EXCEPTION(3); + FAULT_INJECTION_SEND_RPC_EXCEPTION(16); ImpalaInternalServiceClient::send_CancelQueryFInstances(params); *send_done = true; - FAULT_INJECTION_RECV_RPC_EXCEPTION(3); + FAULT_INJECTION_RECV_RPC_EXCEPTION(16); ImpalaInternalServiceClient::recv_CancelQueryFInstances(_return); } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fb8ea6a9/tests/custom_cluster/test_rpc_exception.py ---------------------------------------------------------------------- diff --git a/tests/custom_cluster/test_rpc_exception.py b/tests/custom_cluster/test_rpc_exception.py index 0c86f45..d05cb3b 100644 --- a/tests/custom_cluster/test_rpc_exception.py +++ b/tests/custom_cluster/test_rpc_exception.py @@ -49,6 +49,8 @@ class TestRPCException(CustomClusterTestSuite): assert result.data == self.EXPECTED_RESULT assert not exception_string except ImpalaBeeswaxException as e: + if exception_string is None: + raise e assert exception_string in str(e) @pytest.mark.execute_serially
