minor, improve odbc performance
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/51f22e28 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/51f22e28 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/51f22e28 Branch: refs/heads/master Commit: 51f22e284467f23c8aeeed8ea79e5133ddd1141e Parents: f32165e Author: lidongsjtu <[email protected]> Authored: Thu Jun 15 17:03:20 2017 +0800 Committer: Hongbin Ma <[email protected]> Committed: Thu Jun 15 21:17:54 2017 +0800 ---------------------------------------------------------------------- odbc/Common/REST.cpp | 67 +++++++++++++++++++++-------------- odbc/Common/REST.h | 1 + odbc/Driver/KO_DIAG.CPP | 17 ++++----- odbc/Driver/KO_EXEC.CPP | 6 ++-- odbc/TestDLL/SimpleQueryTest.cpp | 6 ++-- 5 files changed, 57 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/51f22e28/odbc/Common/REST.cpp ---------------------------------------------------------------------- diff --git a/odbc/Common/REST.cpp b/odbc/Common/REST.cpp index 858ecd4..8f0c7bb 100644 --- a/odbc/Common/REST.cpp +++ b/odbc/Common/REST.cpp @@ -280,34 +280,28 @@ std::unique_ptr <MetadataResponse> restGetMeta ( char* serverAddr, long port, ch wstring cookQuery ( wchar_t* p ) { - wchar_t* q = new wchar_t[wcslen ( p ) + 1]; - wcscpy ( q, p ); - - for ( int i = 0; i < ( int ) wcslen ( q ); i++ ) - { - if ( q[i] == '\r' || q[i] == '\n' || q[i] == '\t' ) - { - q[i] = ' '; - } - } - - wstring ret ( q ); - delete[] q; - size_t pos = 0; + std::wstringstream wss; - for ( size_t pos = 0;; pos += 2 ) + int l = wcslen ( p ); + for ( int i = 0; i < l; i++ ) { - pos = ret . find ( L"\"", pos ); - - if ( pos == wstring::npos ) + if ( p[i] == L'\r' || p[i] == L'\n' || p[i] == L'\t' ) { - break; - } - - ret . insert ( pos, L"\\" ); + wss << L' '; + } + + else if (p[i] == L'"') + { + wss << L"\\\""; + } + + else + { + wss << p[i]; + } } - return ret; + return wss.str(); } wstring getBodyString ( http_response& response ) @@ -389,10 +383,15 @@ std::unique_ptr <SQLResponse> convertToSQLResponse ( int statusFlag, wstring requestQuery ( wchar_t* rawSql, char* serverAddr, long port, char* username, char* passwd, char* project, + bool isPrepare, int* statusFlag) { //using local cache to intercept probing queries - const wchar_t* cachedQueryRes = loadCache ( rawSql ); + const wchar_t* cachedQueryRes = NULL; + + if (isPrepare) { + cachedQueryRes = loadCache ( rawSql ); + } if ( cachedQueryRes != NULL ) { @@ -404,8 +403,24 @@ wstring requestQuery ( wchar_t* rawSql, char* serverAddr, long port, char* usern wstring serverAddrW = completeServerStr ( serverAddr, port ); http_client_config config; config . set_timeout ( utility::seconds ( 36000 ) ); + + //uncomment these lines for debug with proxy + //wstring p = L"http://127.0.0.1:8888";; + //config.set_proxy(web_proxy(p)); + http_client session ( serverAddrW, config ); - http_request request = makeRequest ( username, passwd, L"/kylin/api/query", methods::POST ); + http_request request; + + if (!isPrepare) + { + request = makeRequest ( username, passwd, L"/kylin/api/query", methods::POST ); + } + + else + { + request = makeRequest ( username, passwd, L"/kylin/api/query/prestate", methods::POST ); + } + wstring sql = cookQuery ( rawSql ); std::wstringstream wss; wss << L"{ \"acceptPartial\": false, \"project\" : \"" << project << L"\", " << " \"sql\" : \"" << sql << L"\" }" ; @@ -444,7 +459,7 @@ wstring requestQuery ( wchar_t* rawSql, char* serverAddr, long port, char* usern wstring ret = getBodyString ( response ); - if (*statusFlag == 1) + if (*statusFlag == 1 && isPrepare) { storeCache(rawSql, ret.c_str()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/51f22e28/odbc/Common/REST.h ---------------------------------------------------------------------- diff --git a/odbc/Common/REST.h b/odbc/Common/REST.h index f70d7d5..60c14a7 100644 --- a/odbc/Common/REST.h +++ b/odbc/Common/REST.h @@ -38,4 +38,5 @@ std::unique_ptr <SQLResponse> convertToSQLResponse ( int status, wstring requestQuery ( wchar_t* rawSql, char* serverAddr, long port, char* username, char* passwd, char* project, + bool isPrepare, int* statusFlag); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/51f22e28/odbc/Driver/KO_DIAG.CPP ---------------------------------------------------------------------- diff --git a/odbc/Driver/KO_DIAG.CPP b/odbc/Driver/KO_DIAG.CPP index ac313e7..def819c 100644 --- a/odbc/Driver/KO_DIAG.CPP +++ b/odbc/Driver/KO_DIAG.CPP @@ -644,7 +644,8 @@ RETCODE SQL_API SQLGetDiagRec ( SQLSMALLINT pHandleType, pODBCDiagRow _SQLPutDiagRow ( pODBCDiag pDiag, StrPtr pFunc, StrPtr pState, Long pNativeErrorCode, StrPtr pMsgArgs, va_list pArgs ) { - Char s[4096]; // arbitary and maximum length of message + // to avoid: TST1003: The message for sqlstate '01000' is longer than SQL_MAX_MESSAGE_LENGTH - 1. + Char s[SQL_MAX_MESSAGE_LENGTH - 1]; // arbitary and maximum length of message pODBCDiagRow l; pODBCDiagRow r; @@ -660,7 +661,7 @@ pODBCDiagRow _SQLPutDiagRow ( pODBCDiag pDiag, StrPtr pFunc, StrPtr pState, Long // check if there is some message if ( pMsgArgs ) { - vsprintf ( s + strlen ( s ), pMsgArgs, pArgs ); + vsnprintf ( s + strlen ( s ), SQL_MAX_MESSAGE_LENGTH - 2 - strlen(s), pMsgArgs, pArgs ); } else @@ -1110,13 +1111,13 @@ void _ODBCLogMsg ( LogLevel level, const char* pMsgArgs, ... ) break; } - time_t now = time ( 0 ); - struct tm tstruct; - char buffer[100]; - tstruct = *localtime ( &now ); - strftime ( buffer, 100, "%Y-%m-%d.%X", &tstruct ); + char ts[24]; + SYSTEMTIME sys; + GetLocalTime( &sys ); + sprintf( ts, "%4d-%02d-%02d %02d:%02d:%02d.%03d", sys.wYear, sys.wMonth, sys.wDay, sys.wHour, sys.wMinute, sys.wSecond, sys.wMilliseconds); + _write ( gLogFile, "[", 1 ); - _write ( gLogFile, buffer, strlen ( buffer ) ); + _write ( gLogFile, ts, strlen ( ts ) ); _write ( gLogFile, "]", 1 ); // MSG PARSING http://git-wip-us.apache.org/repos/asf/kylin/blob/51f22e28/odbc/Driver/KO_EXEC.CPP ---------------------------------------------------------------------- diff --git a/odbc/Driver/KO_EXEC.CPP b/odbc/Driver/KO_EXEC.CPP index 514375e..8a8992c 100644 --- a/odbc/Driver/KO_EXEC.CPP +++ b/odbc/Driver/KO_EXEC.CPP @@ -129,7 +129,7 @@ RETCODE SQL_API _SQLExecStmtFromReq ( pODBCStmt pStmt, bool pPrepared ) { try { __ODBCLOG ( _ODBCLogMsg ( LogLevel_DEBUG, "start to call rest api" ) ); response = requestQuery ( pStmt->Stmt, pStmt->Conn->Server, pStmt->Conn->ServerPort, pStmt->Conn->UserName, pStmt->Conn->Password, - pStmt->Conn->Project, &status ); + pStmt->Conn->Project, pPrepared, &status ); __ODBCLOG ( _ODBCLogMsg ( LogLevel_DEBUG, "received and uncompressed rest response:" ) ); p = convertToSQLResponse(status, response); __ODBCLOG ( _ODBCLogMsg ( LogLevel_DEBUG, "parsed to SQLResponse" ) ); @@ -207,7 +207,7 @@ RETCODE SQL_API SQLPrepareW ( SQLHSTMT pStmt, ( ( pODBCStmt ) pStmt )->Prepared = 1; - return _SQLExecStmtFromReq((pODBCStmt)pStmt,1); + return _SQLExecStmtFromReq((pODBCStmt)pStmt, 1); //return SQL_SUCCESS; } @@ -230,7 +230,7 @@ RETCODE SQL_API SQLExecute ( HSTMT pStmt ) { } // excute the request - x = _SQLExecStmtFromReq ( ( pODBCStmt ) pStmt, 1 ); + x = _SQLExecStmtFromReq ( ( pODBCStmt ) pStmt, 0 ); return x; } http://git-wip-us.apache.org/repos/asf/kylin/blob/51f22e28/odbc/TestDLL/SimpleQueryTest.cpp ---------------------------------------------------------------------- diff --git a/odbc/TestDLL/SimpleQueryTest.cpp b/odbc/TestDLL/SimpleQueryTest.cpp index 7f857a9..9705b5e 100644 --- a/odbc/TestDLL/SimpleQueryTest.cpp +++ b/odbc/TestDLL/SimpleQueryTest.cpp @@ -23,7 +23,7 @@ void simpleQueryTest () //Intercept query test { int status; - wstring s = requestQuery ( L"SELECT 1", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, &status ); + wstring s = requestQuery ( L"SELECT 1", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, false, &status ); std::unique_ptr <SQLResponse> y = convertToSQLResponse(status, s); if ( ( int ) y -> results . size () != 1 ) @@ -34,7 +34,7 @@ void simpleQueryTest () //Ungzipped Query Test { int status; - wstring s = requestQuery ( L"select cal_dt from test_kylin_fact limit 1", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, &status ); + wstring s = requestQuery ( L"select cal_dt from test_kylin_fact limit 1", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, false, &status ); std::unique_ptr <SQLResponse> y = convertToSQLResponse(status, s); if ( ( int ) y -> results . size () != 1 ) @@ -45,7 +45,7 @@ void simpleQueryTest () //zipped Query Test { int status; - wstring s = requestQuery ( L"select * from test_kylin_fact limit 12", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, &status ); + wstring s = requestQuery ( L"select * from test_kylin_fact limit 12", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, false, &status ); std::unique_ptr <SQLResponse> y = convertToSQLResponse(status, s); if ( ( int ) y -> results . size () != 12 )
