HAWQ-539. Add fault injection for dispatcher QD site
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/1a75a2a7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/1a75a2a7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/1a75a2a7 Branch: refs/heads/HAWQ-459 Commit: 1a75a2a7c191ce16371d9e41d62786228dfa5774 Parents: 4d1a854 Author: Chunling Wang <[email protected]> Authored: Tue Mar 15 10:15:03 2016 +0800 Committer: ivan <[email protected]> Committed: Wed Mar 16 14:09:11 2016 +0800 ---------------------------------------------------------------------- src/backend/cdb/cdbdispatchresult.c | 9 +++++++++ src/backend/cdb/dispatcher.c | 9 +++++++++ src/backend/cdb/workermgr.c | 10 ++++++++++ src/backend/utils/misc/faultinjector.c | 9 ++++++++- src/include/utils/faultinjector.h | 2 ++ tools/bin/gppylib/programs/clsInjectFault.py | 5 ++++- tools/bin/hawqpylib/programs/clsInjectFault.py | 5 ++++- 7 files changed, 46 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/src/backend/cdb/cdbdispatchresult.c ---------------------------------------------------------------------- diff --git a/src/backend/cdb/cdbdispatchresult.c b/src/backend/cdb/cdbdispatchresult.c index 9ba90c0..d97dda5 100644 --- a/src/backend/cdb/cdbdispatchresult.c +++ b/src/backend/cdb/cdbdispatchresult.c @@ -33,6 +33,7 @@ #include "lib/stringinfo.h" /* StringInfoData */ #include "utils/guc.h" /* log_min_messages */ +#include "utils/faultinjector.h" #include "cdb/cdbconn.h" /* SegmentDatabaseDescriptor */ #include "cdb/cdbpartition.h" @@ -110,6 +111,14 @@ cdbdisp_makeResult(struct CdbDispatchResults *meleeResults, meleeIndex = meleeResults->resultCount++; dispatchResult = &meleeResults->resultArray[meleeIndex]; +#ifdef FAULT_INJECTOR + FaultInjector_InjectFaultIfSet( + CreateCdbDispathResultObject, + DDLNotSpecified, + "", // databaseName + ""); // tableName +#endif + /* Initialize CdbDispatchResult. */ dispatchResult->meleeResults = meleeResults; dispatchResult->meleeIndex = meleeIndex; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/src/backend/cdb/dispatcher.c ---------------------------------------------------------------------- diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c index 20e97ab..5d7105c 100644 --- a/src/backend/cdb/dispatcher.c +++ b/src/backend/cdb/dispatcher.c @@ -49,6 +49,7 @@ #include "utils/memutils.h" /* GetMemoryChunkContext */ #include "cdb/cdbsrlz.h" /* serializeNode */ #include "utils/datum.h" /* datumGetSize */ +#include "utils/faultinjector.h" #include "utils/lsyscache.h" /* get_typlenbyval */ #include "miscadmin.h" /* CHECK_FOR_INTERRUPTS */ #include "tcop/tcopprot.h" /* ResetUsage */ @@ -963,6 +964,14 @@ dispatcher_bind_executor(DispatchData *data) continue; } +#ifdef FAULT_INJECTOR + FaultInjector_InjectFaultIfSet( + ConnectionFailAfterGangCreation, + DDLNotSpecified, + "", // databaseName + ""); // tableName +#endif + if (!executormgr_bind_executor_task(data, executor, desc, task, slice)) return false; data->num_of_cached_executors++; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/src/backend/cdb/workermgr.c ---------------------------------------------------------------------- diff --git a/src/backend/cdb/workermgr.c b/src/backend/cdb/workermgr.c index eaf3060..2a31d29 100644 --- a/src/backend/cdb/workermgr.c +++ b/src/backend/cdb/workermgr.c @@ -35,6 +35,8 @@ #include "cdb/cdbgang.h" /* gp_pthread_create */ #include "miscadmin.h" /* TODO: InterruptPending */ +#include "utils/faultinjector.h" + /* * This structure abstract the general job. @@ -114,6 +116,14 @@ workermgr_submit_job(WorkerMgrState *state, i++; worker_mgr_thread->func = func; +#ifdef FAULT_INJECTOR + FaultInjector_InjectFaultIfSet( + WorkerManagerSubmitJob, + DDLNotSpecified, + "", // databaseName + ""); // tableName +#endif + worker_mgr_thread->thread_ret = gp_pthread_create(&worker_mgr_thread->thread, workermgr_thread_func, worker_mgr_thread, "submit_plan_to_qe"); if (worker_mgr_thread->thread_ret) goto error_cleanup; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/src/backend/utils/misc/faultinjector.c ---------------------------------------------------------------------- diff --git a/src/backend/utils/misc/faultinjector.c b/src/backend/utils/misc/faultinjector.c index 44b7462..f5ab5bc 100644 --- a/src/backend/utils/misc/faultinjector.c +++ b/src/backend/utils/misc/faultinjector.c @@ -281,8 +281,12 @@ FaultInjectorIdentifierEnumToString[] = { /* inject fault after dispatcher thread creation*/ /* _("dispatch_wait"), inject fault after dispatcher wait for results from segments*/ - _("connection_fail_after_gang"), + _("connection_fail_after_gang_creation"), /* inject fault after gang thread creation, set connection null*/ + _("create_cdb_dispath_result_object"), + /* inject fault when create cdb dispatch result object, set out of memory */ + _("worker_manager_submit_job"), + /* inject fault when worker manager submit job , set error*/ /* _("make_dispatch_thread"), inject fault when initialing memory structure for dispatcher thread*/ _("before_dispatch"), @@ -1070,6 +1074,9 @@ FaultInjector_NewHashEntry( case DtmXLogDistributedCommit: case AnalyzeSubxactError: case OptTaskAllocateStringBuffer: + case ConnectionFailAfterGangCreation: + case CreateCdbDispathResultObject: + case WorkerManagerSubmitJob: /* These faults are designed for master. */ if(!AmIMaster()) http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/src/include/utils/faultinjector.h ---------------------------------------------------------------------- diff --git a/src/include/utils/faultinjector.h b/src/include/utils/faultinjector.h index f197d2e..ca9c278 100644 --- a/src/include/utils/faultinjector.h +++ b/src/include/utils/faultinjector.h @@ -188,6 +188,8 @@ typedef enum FaultInjectorIdentifier_e { ConnectionFailAfterGangCreation, /* MakeDispatchThread,*/ + CreateCdbDispathResultObject, + WorkerManagerSubmitJob, BeforeDispatch, http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/tools/bin/gppylib/programs/clsInjectFault.py ---------------------------------------------------------------------- diff --git a/tools/bin/gppylib/programs/clsInjectFault.py b/tools/bin/gppylib/programs/clsInjectFault.py index be6d5ce..6f097d8 100644 --- a/tools/bin/gppylib/programs/clsInjectFault.py +++ b/tools/bin/gppylib/programs/clsInjectFault.py @@ -405,7 +405,10 @@ class GpInjectFaultProgram: "analyze_subxact_error (inject an error during analyze)," \ "opt_task_allocate_string_buffer (inject fault while allocating string buffer), " \ "runaway_cleanup (inject fault before starting the cleanup for a runaway query)" \ - "all (affects all faults injected, used for 'status' and 'reset'), ") + "connection_fail_after_gang_creation (inject fault after gang thread creation, set connection null)" \ + "create_cdb_dispath_result_object (inject fault when create cdb dispatch result object, set out of memeory)" \ + "worker_manager_submit_job (inject fault when worker manager submit job , set error)" \ + "all (affects all faults injected, used for 'status' and 'reset'), ") addTo.add_option("-c", "--ddl_statement", dest="ddlStatement", type="string", metavar="ddlStatement", help="The DDL statement on which fault should be set and triggered " \ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/tools/bin/hawqpylib/programs/clsInjectFault.py ---------------------------------------------------------------------- diff --git a/tools/bin/hawqpylib/programs/clsInjectFault.py b/tools/bin/hawqpylib/programs/clsInjectFault.py index af4d9df..8992f9d 100644 --- a/tools/bin/hawqpylib/programs/clsInjectFault.py +++ b/tools/bin/hawqpylib/programs/clsInjectFault.py @@ -413,7 +413,10 @@ class HAWQInjectFaultProgram: "analyze_subxact_error (inject an error during analyze)," \ "opt_task_allocate_string_buffer (inject fault while allocating string buffer), " \ "runaway_cleanup (inject fault before starting the cleanup for a runaway query)" \ - "all (affects all faults injected, used for 'status' and 'reset'), ") + "connection_fail_after_gang_creation (inject fault after gang thread creation, set connection null)" \ + "create_cdb_dispath_result_object (inject fault when create cdb dispatch result object, set out of memory)" \ + "worker_manager_submit_job (inject fault when worker manager submit job , set error)" \ + "all (affects all faults injected, used for 'status' and 'reset'), ") addTo.add_option("-c", "--ddl_statement", dest="ddlStatement", type="string", metavar="ddlStatement", help="The DDL statement on which fault should be set and triggered " \
