Repository: incubator-trafodion Updated Branches: refs/heads/master 06d26c4d6 -> 51f8743da
[TRAFODION-2433] Capture and log call trace upon internal assert failure. This closes #909 Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/51f8743d Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/51f8743d Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/51f8743d Branch: refs/heads/master Commit: 51f8743da24eac861e5170f864f522e22843196c Parents: 06d26c4 Author: Prashant Vasudev <prashanth.vasu...@esgyn.com> Authored: Tue Jan 10 17:25:49 2017 +0000 Committer: selvaganesang <se...@apache.org> Committed: Thu Feb 2 03:18:21 2017 +0000 ---------------------------------------------------------------------- core/sql/arkcmp/CompException.cpp | 82 ++++++++++---- core/sql/arkcmp/CompException.h | 25 +++-- core/sql/executor/ex_control.cpp | 8 +- core/sql/export/ExceptionCallBack.h | 6 +- core/sql/export/NAAbort.cpp | 14 ++- core/sql/nskgmake/Makerules.linux | 4 +- core/sql/sqlcomp/CmpMain.cpp | 15 +++ core/sql/sqlmxevents/logmxevent_traf.cpp | 150 +++++++++++++++++--------- core/sql/sqlmxevents/logmxevent_traf.h | 7 +- 9 files changed, 225 insertions(+), 86 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51f8743d/core/sql/arkcmp/CompException.cpp ---------------------------------------------------------------------- diff --git a/core/sql/arkcmp/CompException.cpp b/core/sql/arkcmp/CompException.cpp index 85ef708..2a73a48 100644 --- a/core/sql/arkcmp/CompException.cpp +++ b/core/sql/arkcmp/CompException.cpp @@ -103,21 +103,36 @@ void DDLException::throwException() FatalException::FatalException(const char *msg, const char *fileName, - UInt32 lineNum) + UInt32 lineNum, + const char *stackTrace) : BaseException(fileName, lineNum) { - if(msg) { - strncpy(msg_, msg, sizeof(msg_)); - msg_[sizeof(msg_)-1] = 0; + int len = 0; + if(msg && (len = strlen(msg))) { + len = MINOF(len,EXCEPTION_MSG_SIZE); + strncpy(msg_, msg, len); + msg_[len] = 0; } else { msg_[0] = 0; } + + if(stackTrace && (len = strlen(stackTrace))) { + len = MINOF(len, STACK_TRACE_SIZE); + strncpy(stackTrace_, stackTrace, len); + stackTrace_[len] = 0; + } + else { + stackTrace_[0] = 0; + } } const char * FatalException::getMsg() { return msg_; } +const char * FatalException::getStackTrace() +{ return stackTrace_; } + void FatalException::throwException() { doFFDCDebugger(msg_); @@ -132,8 +147,8 @@ CmpInternalException::CmpInternalException(const char *msg, : BaseException(fileName, lineNum) { if(msg) { - strncpy(msg_, msg, sizeof(msg_)); - msg_[sizeof(msg_)-1] = 0; + strncpy(msg_, msg, EXCEPTION_MSG_SIZE); + msg_[EXCEPTION_MSG_SIZE-1] = 0; } else { msg_[0] = 0; @@ -153,34 +168,61 @@ void CmpInternalException::throwException() AssertException::AssertException(const char *condition, const char *fileName, - UInt32 lineNum) + UInt32 lineNum, + const char *stackTrace) : BaseException(fileName, lineNum) { - if(condition) { - strncpy(condition_, condition, sizeof(condition_)); - condition_[sizeof(condition_)-1] = 0; + int len = 0; + if(condition && (len = strlen(condition))) { + len = MINOF(len ,EXCEPTION_CONDITION_SIZE); + strncpy(condition_, condition, len); + condition_[len] = 0; } else { condition_[0] = 0; } + + if(stackTrace && (len = strlen(stackTrace))) { + len = MINOF(len, STACK_TRACE_SIZE); + strncpy(stackTrace_, stackTrace, len ); + stackTrace_[len] = 0; + } + else { + stackTrace_[0] = 0; + } } AssertException::AssertException(AssertException & e) : BaseException(e.getFileName(), e.getLineNum()) { + int len = 0; const char *condition = e.getCondition(); - if(condition) { - strncpy(condition_, condition, sizeof(condition_)); - condition_[sizeof(condition_)-1] = 0; + if(condition && (len = strlen(condition))) { + len = MINOF(len , EXCEPTION_CONDITION_SIZE); + strncpy(condition_, condition, len); + condition_[len] = 0; } else { condition_[0] = 0; } + + const char *stackTrace = e.getStackTrace(); + if(stackTrace && (len = strlen(stackTrace))) { + len = MINOF(len, STACK_TRACE_SIZE); + strncpy(stackTrace_, stackTrace, len); + stackTrace_[len] = 0; + } + else { + stackTrace_[0] = 0; + } } const char * AssertException::getCondition() { return condition_; } +const char * AssertException::getStackTrace() +{ return stackTrace_; } + void AssertException::throwException() { doFFDCDebugger(condition_); @@ -213,8 +255,8 @@ PassOneAssertFatalException::PassOneAssertFatalException(const char *condition, lineNum) { if(condition) { - strncpy(condition_, condition, sizeof(condition_)); - condition_[sizeof(condition_)-1] = 0; + strncpy(condition_, condition, EXCEPTION_CONDITION_SIZE); + condition_[EXCEPTION_CONDITION_SIZE-1] = 0; } else { condition_[0] = 0; @@ -270,13 +312,15 @@ void OptAssertException::throwException() void CmpExceptionCallBack::throwFatalException(const char *msg, const char *file, - UInt32 line) -{ FatalException(msg, file, line).throwException(); } + UInt32 line, + const char *stackTrace) +{ FatalException(msg, file, line, stackTrace).throwException(); } void CmpExceptionCallBack::throwAssertException(const char *cond, const char *file, - UInt32 line) -{ AssertException(cond, file, line).throwException(); } + UInt32 line, + const char *stackTrace) +{ AssertException(cond, file, line, stackTrace).throwException(); } CmpExceptionCallBack CmpExceptionEnv::eCallBack_; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51f8743d/core/sql/arkcmp/CompException.h ---------------------------------------------------------------------- diff --git a/core/sql/arkcmp/CompException.h b/core/sql/arkcmp/CompException.h index b919953..afafb7a 100644 --- a/core/sql/arkcmp/CompException.h +++ b/core/sql/arkcmp/CompException.h @@ -38,6 +38,7 @@ #include "NABoolean.h" #include "NABasicObject.h" #include "ExceptionCallBack.h" +#include "logmxevent.h" // BaseException should not be instantiated directly class BaseException : public NABasicObject @@ -78,11 +79,14 @@ public: // FatalException is unrecoverable, give up the compilation if one is thrown class FatalException : public BaseException{ public: - FatalException(const char *msg, const char *fileName, UInt32 lineNum); + FatalException(const char *msg, const char *fileName, UInt32 lineNum, + const char *stackTrace = NULL); const char * getMsg(); + const char * getStackTrace(); virtual void throwException(); private: - char msg_[256]; + char msg_[EXCEPTION_MSG_SIZE]; + char stackTrace_[STACK_TRACE_SIZE]; }; // CmpInternalException is a replacement for EH_INTRNAL_EXCEPTION @@ -92,7 +96,7 @@ public: const char * getMsg(); virtual void throwException(); private: - char msg_[256]; + char msg_[EXCEPTION_MSG_SIZE]; }; // AssertException is thrown from an Assertion in the compiler. @@ -101,13 +105,16 @@ class AssertException : public BaseException{ public: AssertException(const char *condition, const char *fileName, - UInt32 lineNum); + UInt32 lineNum, + const char *stackTrace = NULL); // copy contructor AssertException(AssertException & e); const char * getCondition(); + const char * getStackTrace(); virtual void throwException(); private: - char condition_[128]; + char condition_[EXCEPTION_CONDITION_SIZE]; + char stackTrace_[STACK_TRACE_SIZE]; }; class OsimLogException : public BaseException{ @@ -128,7 +135,7 @@ public: UInt32 lineNum); virtual void throwException(); private: - char condition_[128]; + char condition_[EXCEPTION_CONDITION_SIZE]; }; class PassOneNoPlanFatalException : public FatalException{ @@ -158,10 +165,12 @@ class CmpExceptionCallBack : public ExceptionCallBack public: void throwFatalException(const char *msg, const char *file, - UInt32 line); + UInt32 line, + const char *stackTrace = NULL); void throwAssertException(const char *cond, const char *file, - UInt32 line); + UInt32 line, + const char *stackTrace = NULL); }; class CmpExceptionEnv{ http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51f8743d/core/sql/executor/ex_control.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ex_control.cpp b/core/sql/executor/ex_control.cpp index 70a1271..5a7ba4e 100644 --- a/core/sql/executor/ex_control.cpp +++ b/core/sql/executor/ex_control.cpp @@ -213,14 +213,16 @@ short ExControlTcb::work() NULL, 0, da); if (cmpStatus != 0) { - char emsText[120]; + char *emsText = new (getHeap()) char[dataLen + 120]; str_sprintf(emsText, - "Set control to embedded arkcmp failed, return code %d", - cmpStatus); + "Set control to embedded arkcmp failed, return code %d, data: %s", + cmpStatus, data); SQLMXLoggingArea::logExecRtInfo(__FILE__, __LINE__, emsText, 0); ExHandleArkcmpErrors(qparent_, pentry_down, 0, getGlobals(), da); + // da->clear(); + getHeap()->deallocateMemory(emsText); } else saveControl = TRUE; // need to save control to exe ControlInfoTable http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51f8743d/core/sql/export/ExceptionCallBack.h ---------------------------------------------------------------------- diff --git a/core/sql/export/ExceptionCallBack.h b/core/sql/export/ExceptionCallBack.h index 3dd8a6d..fa90d7d 100644 --- a/core/sql/export/ExceptionCallBack.h +++ b/core/sql/export/ExceptionCallBack.h @@ -38,10 +38,12 @@ class ExceptionCallBack { public: virtual void throwAssertException(const char * condition, const char * fileName, - UInt32 lineNum) = 0; + UInt32 lineNum, + const char *stackTrace=NULL) = 0; virtual void throwFatalException(const char * msg, const char * fileName, - UInt32 lineNum) = 0; + UInt32 lineNum, + const char *stackTrace=NULL) = 0; }; #endif http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51f8743d/core/sql/export/NAAbort.cpp ---------------------------------------------------------------------- diff --git a/core/sql/export/NAAbort.cpp b/core/sql/export/NAAbort.cpp index 9e539c5..166a640 100644 --- a/core/sql/export/NAAbort.cpp +++ b/core/sql/export/NAAbort.cpp @@ -45,6 +45,8 @@ #include "CompException.h" #include "StmtCompilationMode.h" +extern Int32 writeStackTrace(char *s, int bufLen); + // Mutex to serialize termination via NAExit or an assertion failure // via abort_botch_abend in the main executor thread with an assertion // failure in the SeaMonster reader thread @@ -150,7 +152,11 @@ void NAInternalError::throwFatalException(const char *msg, UInt32 line) { if(pExceptionCallBack_ != NULL) - pExceptionCallBack_->throwFatalException(msg, file, line); + { + char stackTrace[STACK_TRACE_SIZE]; + writeStackTrace(stackTrace, sizeof(stackTrace)); + pExceptionCallBack_->throwFatalException(msg, file, line, stackTrace); + } } void NAInternalError::throwAssertException(const char *cond, @@ -158,7 +164,11 @@ void NAInternalError::throwAssertException(const char *cond, UInt32 line) { if(pExceptionCallBack_ != NULL) - pExceptionCallBack_->throwAssertException(cond, file, line); + { + char stackTrace[STACK_TRACE_SIZE]; + writeStackTrace(stackTrace, sizeof(stackTrace)); + pExceptionCallBack_->throwAssertException(cond, file, line, stackTrace); + } } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51f8743d/core/sql/nskgmake/Makerules.linux ---------------------------------------------------------------------- diff --git a/core/sql/nskgmake/Makerules.linux b/core/sql/nskgmake/Makerules.linux index 039dd80..802f670 100755 --- a/core/sql/nskgmake/Makerules.linux +++ b/core/sql/nskgmake/Makerules.linux @@ -257,7 +257,9 @@ CFLAGS += -fshort-wchar # same time. # -g3 : Highest level of debugging # -ggdb : Use native debugging format with gdb extensions. - SYS_CXXFLAGS += -fno-strict-aliasing -msse2 -mfpmath=sse -g3 -ggdb + # -rdynamic : Adds dynamic linker symbols to get the stack trace + # of a running program. Note these are not debug symbols. + SYS_CXXFLAGS += -fno-strict-aliasing -msse2 -mfpmath=sse -g3 -ggdb -rdynamic CFLAGS += -fno-strict-aliasing -msse2 -mfpmath=sse -g3 -ggdb # Some more definitions are added when building for a 64-bit target. http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51f8743d/core/sql/sqlcomp/CmpMain.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpMain.cpp b/core/sql/sqlcomp/CmpMain.cpp index b83ce79..0b884dc 100644 --- a/core/sql/sqlcomp/CmpMain.cpp +++ b/core/sql/sqlcomp/CmpMain.cpp @@ -1728,6 +1728,13 @@ CmpMain::ReturnStatus CmpMain::sqlcomp(const char *input_str, //IN << DgString0(e.getMsg()) << DgString1(e.getFileName()) << DgInt0((Lng32)e.getLineNum()); + if(e.getStackTrace()) + SQLMXLoggingArea::logSQLMXAssertionFailureEvent(e.getFileName(), + e.getLineNum(), + "Compiler Fatal Exception", + e.getMsg(), + NULL, + e.getStackTrace()); } } catch(AssertException & e){ @@ -1735,6 +1742,14 @@ CmpMain::ReturnStatus CmpMain::sqlcomp(const char *input_str, //IN << DgString0(e.getCondition()) << DgString1(e.getFileName()) << DgInt0((Lng32)e.getLineNum()); + if(e.getStackTrace()) + SQLMXLoggingArea::logSQLMXAssertionFailureEvent(e.getFileName(), + e.getLineNum(), + "Compiler Internal Assert Failure", + e.getCondition(), + NULL, + e.getStackTrace()); + } catch(BaseException & e){ // Probably not reach here, just a safe net http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51f8743d/core/sql/sqlmxevents/logmxevent_traf.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlmxevents/logmxevent_traf.cpp b/core/sql/sqlmxevents/logmxevent_traf.cpp index 2287de7..5813059 100644 --- a/core/sql/sqlmxevents/logmxevent_traf.cpp +++ b/core/sql/sqlmxevents/logmxevent_traf.cpp @@ -42,6 +42,8 @@ #include <limits.h> #include <stdarg.h> #include <execinfo.h> +#include <cxxabi.h> + #ifdef _MSC_VER #undef _MSC_VER #endif @@ -185,71 +187,107 @@ void SQLMXLoggingArea::logPrivMgrInfo(const char *filename, } -static void writeStackTrace(char *s, int bufLen) +Int32 writeStackTrace(char *s, int bufLen) { - const int safetyMargin = 256; - int len = sprintf(s, "Process Stack Trace:\n"); - - // This is a quick and dirty implementation for Linux. It is easy to - // get the program counters for the stack trace, but is difficult to - // look up the function name, line, and file number based off of the - // program counter. For simplicity, this code just calls addr2line to - // look up the information. This could be changed in the future if an - // easy to use API becomes available. void *bt[20]; + char **strings; + size_t funcsize = 256; + size_t newfuncsize = funcsize; + const int safetyMargin = 128; size_t size = backtrace(bt, 20); + strings = backtrace_symbols (bt, size); - pid_t myPID = getpid(); + if(bufLen <= safetyMargin) + return 0; - // Write each level of the stack except for the top frame and the - // bottom two frames, which aren't important here. - Int32 i = 1; - while (i < size - 2) - { - char buffer[128]; // Used for command-line + addr2line output. - char addrBuf[sizeof(void *)*2 + 4]; + //Note this string should fit within safetyMargin size. + int len = sprintf(s, "Process Stack Trace:\n"); - sprintf(buffer, "/usr/bin/addr2line -e /proc/%d/exe -f -C ", myPID); + + // allocate string which will be filled with the demangled function name + // malloc needed since demangle function uses realloc. + + char* funcname = (char*)malloc(funcsize); - Int32 j; + // iterate over the returned symbol lines. skip the first, it is the + // address of this function. + for (int i = 1; i < size; i++) + { + char *begin_name = 0, *begin_offset = 0, *end_offset = 0; - // Run addr2line on 5 addresses at a time. - for (j = i; j < i+5 && j < size-2; j++) + // find parentheses and +address offset surrounding the mangled name: + // ./module(function+0x15c) [0x8048a6d] + for (char *p = strings[i]; *p; ++p) { - sprintf(addrBuf, " %p", bt[j]); - strcat(buffer, addrBuf); + if (*p == '(') + begin_name = p; + else if (*p == '+') + begin_offset = p; + else if (*p == ')' && begin_offset) + { + end_offset = p; + break; + } } - FILE *cmdFP = popen(buffer, "r"); - if (cmdFP == NULL) + if (begin_name && begin_offset && end_offset + && begin_name < begin_offset) { - if (len+safetyMargin < bufLen) - len += sprintf(s, "Error %d while popen() of %s\n", errno, buffer); - break; + *begin_name++ = '\0'; + *begin_offset++ = '\0'; + *end_offset = '\0'; + + int status; + char* ret = abi::__cxa_demangle(begin_name, + funcname, &newfuncsize, &status); + if (status == 0) + { + funcname = ret; + if((len + strlen(strings[i]) + newfuncsize + safetyMargin) < bufLen) + { + len += sprintf(s + len, "%s : %s+%s\n", + strings[i], funcname, begin_offset); + } + else + { + break; //bufLen is short. Break out. + } + + } + else + { + //demangling failed. just store begin name and offset as is. + if((len + strlen(strings[i]) + strlen(begin_name) + safetyMargin) + < bufLen) + { + len += sprintf(s + len, " %s : %s+%s\n", + strings[i], begin_name, begin_offset); + } + else + { + break; //bufLen is short. Break out. + } + + } } else { - for (j = i; j < i+5 && j < size-2; j++) + // couldn't parse the line. Do nothing. Move to next line. + if((len + strlen(strings[i]) + safetyMargin) < bufLen) { - // Read from the addr2line output - fgets(buffer, sizeof(buffer), cmdFP); - - // Replace newline with null character - size_t len = strlen(buffer); - if (buffer[len-1] == '\n') - buffer[len-1] = '\0'; - - if (len+safetyMargin < bufLen) - len += sprintf(s, "%p: %s()\n", bt[j], buffer); - fgets(buffer, sizeof(buffer), cmdFP); - if (len+safetyMargin < bufLen) - len += sprintf(s, " %s", buffer); + len += sprintf(s + len, " %s\n", strings[i]); + } + else + { + break; //bufLen is short. Break out. } - fclose(cmdFP); } - i = j; } - sprintf(s, "\n"); + + free(funcname); + free(strings); + len += sprintf(s + len, "\n"); //safetyMargin assumed. + return len; } void SQLMXLoggingArea::logErr97Event(int rc) @@ -306,12 +344,13 @@ SQLMXLoggingArea::logSQLMXAbortEvent(const char* file, Int32 line, const char* m // log an ASSERTION FAILURE event void -SQLMXLoggingArea::logSQLMXAssertionFailureEvent(const char* file, Int32 line, const char* msgTxt, const char* condition, const Lng32* tid) +SQLMXLoggingArea::logSQLMXAssertionFailureEvent(const char* file, Int32 line, const char* msgTxt, const char* condition, const Lng32* tid + , const char* stackTrace) { bool lockedMutex = lockMutex(); - Int32 LEN = 8192; - char msg[8192]; + Int32 LEN = SQLEVENT_BUF_SIZE + STACK_TRACE_SIZE; + char msg[LEN]; memset(msg, 0, LEN); Int32 sLen = str_len(msgTxt); @@ -342,8 +381,19 @@ SQLMXLoggingArea::logSQLMXAssertionFailureEvent(const char* file, Int32 line, co str_sprintf(condStr, " CONDITION: %s ", condition); sLen = str_len(condStr); str_cpy_all (msg+sTotalLen, condStr, sLen); + sTotalLen += sLen; } - + + if(stackTrace) + { + str_ncpy(msg+sTotalLen, stackTrace, LEN - sTotalLen); + } + else + { + if((LEN - sTotalLen) > 512 ) + writeStackTrace(msg, LEN - sTotalLen); + } + QRLogger::log(QRLogger::instance().getMyDefaultCat(), LL_FATAL, "%s", msg); if (lockedMutex) http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51f8743d/core/sql/sqlmxevents/logmxevent_traf.h ---------------------------------------------------------------------- diff --git a/core/sql/sqlmxevents/logmxevent_traf.h b/core/sql/sqlmxevents/logmxevent_traf.h index 3303a43..1432535 100644 --- a/core/sql/sqlmxevents/logmxevent_traf.h +++ b/core/sql/sqlmxevents/logmxevent_traf.h @@ -49,6 +49,10 @@ #define EVENTSAPI #define SQLEVENT_BUF_SIZE 4024 +#define STACK_TRACE_SIZE 8192 +#define EXCEPTION_MSG_SIZE 256 +#define EXCEPTION_CONDITION_SIZE 128 + #ifdef _MSC_VER #undef _MSC_VER @@ -130,7 +134,8 @@ class SQLMXLoggingArea Int32 lineno, const char* msg, const char* condition = NULL, - const Lng32* tid = NULL); + const Lng32* tid = NULL, + const char* stackTrace = NULL); //TBD