Repository: incubator-trafodion Updated Branches: refs/heads/master a89a6585a -> 660c80e33
[TRAFODION-2440] Replace fixed length buffers with strings in LOB code Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/49498dfd Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/49498dfd Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/49498dfd Branch: refs/heads/master Commit: 49498dfdf6b629bf349f9630fdb5ceb197e93bf9 Parents: 8c7ff9a Author: Dave Birdsall <[email protected]> Authored: Tue Jan 17 17:07:11 2017 +0000 Committer: Dave Birdsall <[email protected]> Committed: Tue Jan 17 17:07:11 2017 +0000 ---------------------------------------------------------------------- core/sql/exp/ExpLOBaccess.cpp | 219 ++++++++++++++++++++----------------- core/sql/exp/ExpLOBaccess.h | 29 ++--- 2 files changed, 137 insertions(+), 111 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/49498dfd/core/sql/exp/ExpLOBaccess.cpp ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp index aecf153..e5bd380 100644 --- a/core/sql/exp/ExpLOBaccess.cpp +++ b/core/sql/exp/ExpLOBaccess.cpp @@ -84,8 +84,7 @@ ExLob::ExLob() : openFlags_(0), lobTrace_(FALSE) { - memset(lobDataFile_,'\0',sizeof(lobDataFile_)); - + // nothing else to do } ExLob::~ExLob() @@ -98,7 +97,7 @@ ExLob::~ExLob() } -Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode, +Ex_Lob_Error ExLob::initialize(const char *lobFile, Ex_Lob_Mode mode, char *lobStorageLocation, LobsStorage storage, char *hdfsServer, Int64 hdfsPort, @@ -119,15 +118,16 @@ Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode, } if (lobFile) - snprintf(lobDataFile_, MAX_LOB_FILE_NAME_LEN, "%s/%s", - lobStorageLocation_.c_str(), - lobFile); - + { + lobDataFile_ = lobStorageLocation; + lobDataFile_ += "/"; + lobDataFile_ += lobFile; + } } else { if (lobFile) - snprintf(lobDataFile_, MAX_LOB_FILE_NAME_LEN, "%s", lobFile); + lobDataFile_ = lobFile; } @@ -167,14 +167,14 @@ Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode, if (mode == EX_LOB_CREATE) { // check if file is already created - hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_); + hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.c_str()); if (fInfo != NULL) { hdfsFreeFileInfo(fInfo, 1); return LOB_DATA_FILE_CREATE_ERROR; } openFlags = O_WRONLY | O_CREAT; - fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags, bufferSize, replication, blockSize); + fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags, bufferSize, replication, blockSize); if (!fdData_) { return LOB_DATA_FILE_CREATE_ERROR; @@ -198,7 +198,7 @@ Ex_Lob_Error ExLob::fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOff lobCursors_it it = lobCursors_.find(string(handleIn, handleLenIn)); char logBuf[4096]; lobDebugInfo("In ExLob::fetchCursor",0,__LINE__,lobTrace_); - char *blackBox = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6]; + char *blackBox = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6]; // TODO: do we ever go this way for Hive? Int32 blackBoxLen = 0; if (it == lobCursors_.end()) @@ -251,8 +251,10 @@ Ex_Lob_Error ExLob::fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOff { // we have received the external data file name from the descriptor table // replace the contents of the lobDataFile with this name - str_cpy_and_null(lobDataFile_, blackBox, blackBoxLen,'\0','0',TRUE); - + lobDataFile_.assign(blackBox,0,blackBoxLen); // copy blackBox safely (up to blackBoxLen chars) + size_t found = lobDataFile_.find_last_not_of("0"); + if (found != string::npos) + lobDataFile_.erase(found+1); // trim off any trailing '0' } outOffset = offset; outSize = size; @@ -304,7 +306,7 @@ Ex_Lob_Error ExLob::writeData(Int64 offset, char *data, Int32 size, Int64 &operL if (!fdData_ || (openFlags_ != (O_WRONLY | O_APPEND))) // file is not open for write { // get file info - hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_); + hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.c_str()); if (fInfo == NULL) { return LOB_DATA_FILE_NOT_FOUND_ERROR; } @@ -312,7 +314,7 @@ Ex_Lob_Error ExLob::writeData(Int64 offset, char *data, Int32 size, Int64 &operL hdfsCloseFile(fs_, fdData_); fdData_=NULL; openFlags_ = O_WRONLY | O_APPEND; - fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0); + fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, 0, 0, 0); if (!fdData_) { openFlags_ = -1; return LOB_DATA_FILE_OPEN_ERROR; @@ -338,7 +340,7 @@ Ex_Lob_Error ExLob::writeDataSimple(char *data, Int64 size, LobsSubOper subOpera if (!fdData_ || (openFlags_ != (O_WRONLY | O_APPEND))) // file is not open for write { // get file info - hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_); + hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.c_str()); if (fInfo == NULL) { return LOB_DATA_FILE_NOT_FOUND_ERROR; } else { @@ -351,7 +353,7 @@ Ex_Lob_Error ExLob::writeDataSimple(char *data, Int64 size, LobsSubOper subOpera hdfsCloseFile(fs_, fdData_); fdData_=NULL; openFlags_ = O_WRONLY | O_APPEND ; - fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, bufferSize, replication, blockSize); + fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, bufferSize, replication, blockSize); if (!fdData_) { openFlags_ = -1; return LOB_DATA_FILE_OPEN_ERROR; @@ -976,8 +978,10 @@ Ex_Lob_Error ExLob::readToMem(char *memAddr, Int64 size, Int64 &operLen,char * // we have received the external data file name from the descriptor table // replace the contents of the lobDataFile with this name - str_cpy_and_null(lobDataFile_, blackBox, blackBoxLen,'\0','0',TRUE); - + lobDataFile_.assign(blackBox,0,blackBoxLen); // copy blackBox safely (up to blackBoxLen chars) + size_t found = lobDataFile_.find_last_not_of("0"); + if (found != string::npos) + lobDataFile_.erase(found+1); // trim off any trailing '0' } if (blackBoxLen == -1) { @@ -1048,12 +1052,14 @@ Ex_Lob_Error ExLob::readToFile(char *tgtFileName, Int64 tgtLength, Int64 &operLe } if (blackBox) - { - // we have received the external data file name from the descriptor table - // replace the contents of the lobDataFile with this name - str_cpy_and_null(lobDataFile_, blackBox, blackBoxLen,'\0','0',TRUE); - - } + { + // we have received the external data file name from the descriptor table + // replace the contents of the lobDataFile with this name + lobDataFile_.assign(blackBox,0,blackBoxLen); // copy blackBox safely (up to blackBoxLen chars) + size_t found = lobDataFile_.find_last_not_of("0"); + if (found != string::npos) + lobDataFile_.erase(found+1); // trim off any trailing '0' + } if (tgtType == HDFS_FILE) { err = readDataToHdfsFile(tgtFileName, srcOffset , tgtLength,operLen, lobMaxChunkMemLen, fileflags,handleIn,handleInLen,multipleChunks,transId); @@ -1281,10 +1287,11 @@ Ex_Lob_Error ExLob::delDesc(char *handleIn, Int32 handleInLen, Int64 transId) Ex_Lob_Error ExLob::purgeLob() { char logBuf[4096]; - if (hdfsDelete(fs_, lobDataFile_, 0) != 0) + if (hdfsDelete(fs_, lobDataFile_.c_str(), 0) != 0) { - str_sprintf(logBuf,"hdfsDelete of %s returned error",lobDataFile_); - lobDebugInfo(lobDataFile_,0,__LINE__,lobTrace_); + string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 40)); // make small enough to fit + str_sprintf(logBuf,"hdfsDelete of %s returned error",lobDataFileSubstr.c_str()); + lobDebugInfo("In ExLob::purgeLob",0,__LINE__,lobTrace_); return LOB_DATA_FILE_DELETE_ERROR; } @@ -1340,7 +1347,7 @@ Ex_Lob_Error ExLob::openCursor(char *handleIn, Int32 handleInLen,Int64 transId) return LOB_OPER_OK; } -Ex_Lob_Error ExLob::openDataCursor(char *file, LobsCursorType type, +Ex_Lob_Error ExLob::openDataCursor(const char *file, LobsCursorType type, Int64 range, Int64 bufMaxSize, Int64 maxBytes, Int64 waited, ExLobGlobals *lobGlobals, @@ -1383,7 +1390,7 @@ Ex_Lob_Error ExLob::openDataCursor(char *file, LobsCursorType type, cursor.prefetch_ = !waited; cursor.bufferHits_ = 0; cursor.bufferMisses_ = 0; - strcpy(cursor.name_, file); + cursor.name_ = file; cursor.currentRange_ = -1; cursor.endRange_ = -1; @@ -1405,7 +1412,7 @@ Ex_Lob_Error ExLob::openDataCursor(char *file, LobsCursorType type, hdfsCloseFile(fs_, fdData_); fdData_ = NULL; openFlags_ = O_RDONLY; - fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0); + fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, 0, 0, 0); if (!fdData_) { @@ -1501,31 +1508,28 @@ Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset, if (size == 0) //we are trying to empty this lob. { //rename lob datafile - char * saveLobDataFile = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6]; - str_sprintf(saveLobDataFile, "%s_save",lobDataFile_); - Int32 rc2 = hdfsRename(fs_,lobDataFile_,saveLobDataFile); + string saveLobDataFile = lobDataFile_ + "_save"; + Int32 rc2 = hdfsRename(fs_,lobDataFile_.c_str(),saveLobDataFile.c_str()); if (rc2 == -1) { lobDebugInfo("Problem renaming datafile to save data file",0,__LINE__,lobTrace_); - NADELETEBASIC(saveLobDataFile,getLobGlobalHeap()); return LOB_DATA_FILE_WRITE_ERROR; } //create a new file of the same name. - hdfsFile fdNew = hdfsOpenFile(fs_, lobDataFile_,O_WRONLY|O_CREAT,0,0,0); + hdfsFile fdNew = hdfsOpenFile(fs_, lobDataFile_.c_str(),O_WRONLY|O_CREAT,0,0,0); if (!fdNew) { - str_sprintf(logBuf,"Could not create/open file:%s",lobDataFile_); + string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 40)); // make small enough to fit + str_sprintf(logBuf,"Could not create/open file:%s",lobDataFileSubstr.c_str()); lobDebugInfo(logBuf,0,__LINE__,lobTrace_); //restore previous version - Int32 rc2 = hdfsRename(fs_,saveLobDataFile,lobDataFile_); + Int32 rc2 = hdfsRename(fs_,saveLobDataFile.c_str(),lobDataFile_.c_str()); if (rc2 == -1) { lobDebugInfo("Problem restoring datafile . Will need to retry the update",0,__LINE__,lobTrace_); - NADELETEBASIC(saveLobDataFile,getLobGlobalHeap()); return LOB_DATA_FILE_WRITE_ERROR; } - NADELETEBASIC(saveLobDataFile,getLobGlobalHeap()); return LOB_DATA_FILE_OPEN_ERROR; } @@ -1533,17 +1537,16 @@ Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset, { //A new empty data file has been created. // delete the saved data file - Int32 rc2 = hdfsDelete(fs_,saveLobDataFile,FALSE);//ok to ignore error.nt32 + Int32 rc2 = hdfsDelete(fs_,saveLobDataFile.c_str(),FALSE);//ok to ignore error.nt32 if (rc2 == -1) { lobDebugInfo("Problem deleting saved datafile . Will need to manually cleanup saved datafile",0,__LINE__,lobTrace_); } - NADELETEBASIC(saveLobDataFile,getLobGlobalHeap()); hdfsCloseFile(fs_,fdNew); fdNew = NULL; } } - hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_); + hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.c_str()); if (fInfo) dataOffset = fInfo->mSize; @@ -1571,17 +1574,19 @@ Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset, if (GCDone) // recalculate the new offset { hdfsFreeFileInfo(fInfo, 1); - fInfo = hdfsGetPathInfo(fs_, lobDataFile_); + fInfo = hdfsGetPathInfo(fs_, lobDataFile_.c_str()); } if (fInfo) dataOffset = fInfo->mSize; + + string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 70)); // make small enough to fit below if (GCDone) str_sprintf(logBuf,"Done GC. Allocating new Offset %Ld in %s", - dataOffset,lobDataFile_); + dataOffset,lobDataFileSubstr.c_str()); else str_sprintf(logBuf,"Allocating new Offset %Ld in %s ", - dataOffset,lobDataFile_); + dataOffset,lobDataFileSubstr.c_str()); lobDebugInfo(logBuf,0,__LINE__,lobTrace_); //Find the last offset in the file // dataOffset = hdfsTell(fs_,fdData_); //commenting out.hdfsTell always returns 0 !! @@ -1594,12 +1599,14 @@ Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int char logBuf[4096]; lobDebugInfo("In ExLob::compactLobDataFile",0,__LINE__,lobTrace_); Int64 maxMemChunk = 1024*1024*1024; //1GB limit for intermediate buffer for transfering data - char * saveLobDataFile = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6]; - str_sprintf(saveLobDataFile, "%s_save",lobDataFile_); - char * tmpLobDataFile = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+5]; - str_sprintf(tmpLobDataFile, "%s_tmp",lobDataFile_); + string saveLobDataFile = lobDataFile_ + "_save"; + string tmpLobDataFile = lobDataFile_ + "_tmp"; - str_sprintf(logBuf,"DataFile %s, TempDataFile : %s, SaveDataFile : %s ",lobDataFile_,tmpLobDataFile, saveLobDataFile); + string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf)/3 - 20); // make small enough to fit + string tmpLobDataFileSubstr = tmpLobDataFile.substr(0,sizeof(logBuf)/3 - 20); + string saveLobDataFileSubstr = saveLobDataFile.substr(0,sizeof(logBuf)/3 - 20); + str_sprintf(logBuf,"DataFile %s, TempDataFile : %s, SaveDataFile : %s ", + lobDataFileSubstr.c_str(),tmpLobDataFileSubstr.c_str(), saveLobDataFileSubstr.c_str()); lobDebugInfo(logBuf,0,__LINE__,lobTrace_); hdfsFS fs = hdfsConnect(hdfsServer_,hdfsPort_); @@ -1607,11 +1614,12 @@ Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int return LOB_DATA_FILE_OPEN_ERROR; - hdfsFile fdData = hdfsOpenFile(fs, lobDataFile_, O_RDONLY, 0, 0,0); + hdfsFile fdData = hdfsOpenFile(fs, lobDataFile_.c_str(), O_RDONLY, 0, 0,0); if (!fdData) { - str_sprintf(logBuf,"Could not open file:%s",lobDataFile_); + string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 40)); // make small enough to fit + str_sprintf(logBuf,"Could not open file:%s",lobDataFileSubstr.c_str()); lobDebugInfo(logBuf,0,__LINE__,lobTrace_); hdfsCloseFile(fs,fdData); fdData = NULL; @@ -1619,10 +1627,11 @@ Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int } - hdfsFile fdTemp = hdfsOpenFile(fs, tmpLobDataFile,O_WRONLY|O_CREAT,0,0,0); + hdfsFile fdTemp = hdfsOpenFile(fs, tmpLobDataFile.c_str(),O_WRONLY|O_CREAT,0,0,0); if (!fdTemp) { - str_sprintf(logBuf,"Could not open file:%s",tmpLobDataFile); + string tmpLobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 40)); // make small enough to fit + str_sprintf(logBuf,"Could not open file:%s",tmpLobDataFileSubstr.c_str()); lobDebugInfo(logBuf,0,__LINE__,lobTrace_); hdfsCloseFile(fs,fdTemp); fdTemp = NULL; @@ -1691,24 +1700,18 @@ Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int //Now save the data file and rename the tempfile to the original datafile - Int32 rc2 = hdfsRename(fs,lobDataFile_,saveLobDataFile); + Int32 rc2 = hdfsRename(fs,lobDataFile_.c_str(),saveLobDataFile.c_str()); if (rc2 == -1) { lobDebugInfo("Problem renaming datafile to save data file",0,__LINE__,lobTrace_); - NADELETEBASIC(saveLobDataFile,getLobGlobalHeap()); - NADELETEBASIC(tmpLobDataFile,getLobGlobalHeap()); return LOB_DATA_FILE_WRITE_ERROR; } - rc2 = hdfsRename(fs,tmpLobDataFile, lobDataFile_); + rc2 = hdfsRename(fs,tmpLobDataFile.c_str(), lobDataFile_.c_str()); if (rc2 == -1) { lobDebugInfo("Problem renaming temp datafile to data file",0,__LINE__,lobTrace_); - NADELETEBASIC(saveLobDataFile,getLobGlobalHeap()); - NADELETEBASIC(tmpLobDataFile,getLobGlobalHeap()); return LOB_DATA_FILE_WRITE_ERROR; } - NADELETEBASIC(saveLobDataFile,getLobGlobalHeap()); - NADELETEBASIC(tmpLobDataFile,getLobGlobalHeap()); return LOB_OPER_OK; } @@ -1720,18 +1723,15 @@ Ex_Lob_Error ExLob::restoreLobDataFile() hdfsFS fs = hdfsConnect(hdfsServer_,hdfsPort_); if (fs == NULL) return LOB_DATA_FILE_OPEN_ERROR; - char * saveLobDataFile = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6]; - str_sprintf(saveLobDataFile, "%s_save",lobDataFile_); - Int32 rc2 = hdfsDelete(fs,lobDataFile_,FALSE);//ok to ignore error. - rc2 = hdfsRename(fs,saveLobDataFile, lobDataFile_); - if (rc2) + string saveLobDataFile = lobDataFile_ + "_save"; + Int32 rc2 = hdfsDelete(fs,lobDataFile_.c_str(),FALSE);//ok to ignore error. + rc2 = hdfsRename(fs,saveLobDataFile.c_str(), lobDataFile_.c_str()); + if (rc2) { lobDebugInfo("Problem renaming savedatafile to data file",0,__LINE__,lobTrace_); - NADELETEBASIC(saveLobDataFile,getLobGlobalHeap()); return LOB_OPER_ERROR; } - NADELETEBASIC(saveLobDataFile,getLobGlobalHeap()); - return rc; + return rc; } @@ -1742,12 +1742,10 @@ Ex_Lob_Error ExLob::purgeBackupLobDataFile() hdfsFS fs = hdfsConnect(hdfsServer_,hdfsPort_); if (fs == NULL) return LOB_DATA_FILE_OPEN_ERROR; - char * saveLobDataFile = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6]; - str_sprintf(saveLobDataFile, "%s_save",lobDataFile_); - Int32 rc2 = hdfsDelete(fs,saveLobDataFile,FALSE);//ok to ignore error. + string saveLobDataFile = lobDataFile_ + "_save"; + Int32 rc2 = hdfsDelete(fs,saveLobDataFile.c_str(),FALSE);//ok to ignore error. - NADELETEBASIC(saveLobDataFile,getLobGlobalHeap()); - return rc; + return rc; } /////////////////////////////////////////////////////////////////////////////// // ExLobDescHeader definitions @@ -1832,7 +1830,7 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I hdfsCloseFile(fs_, fdData_); fdData_=NULL; openFlags_ = O_RDONLY; - fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0); + fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, 0, 0, 0); if (!fdData_) { @@ -1907,7 +1905,7 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr, hdfsCloseFile(fs_, fdData_); fdData_=NULL; openFlags_ = O_RDONLY; - fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0); + fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, 0, 0, 0); if (!fdData_) { @@ -1919,7 +1917,7 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr, } else { - fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0); + fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, 0, 0, 0); if (!fdData_) { @@ -1940,7 +1938,10 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr, return LOB_DATA_READ_ERROR; } - str_sprintf(logBuf,"After hdfsPread: File:%s, Offset:%Ld, Size:%Ld,Target Mem Addr:%Ld",lobDataFile_,offset,size,memAddr); + + string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 100)); // make small enough to fit + str_sprintf(logBuf,"After hdfsPread: File:%s, Offset:%Ld, Size:%Ld,Target Mem Addr:%Ld", + lobDataFileSubstr.c_str(),offset,size,memAddr); lobDebugInfo(logBuf,0,__LINE__,lobTrace_); operLen = bytesRead; return LOB_OPER_OK; @@ -2285,7 +2286,7 @@ Ex_Lob_Error ExLobsOper ( { Ex_Lob_Error err = LOB_OPER_OK; ExLob *lobPtr = NULL; - char fn[MAX_LOB_FILE_NAME_LEN]; + string fn; struct timespec startTime; struct timespec endTime; Int64 secs, nsecs, totalnsecs; @@ -2300,7 +2301,7 @@ Ex_Lob_Error ExLobsOper ( clock_gettime(CLOCK_MONOTONIC, &startTime); - char *fileName = lobName; + const char *fileName = lobName; if (globPtr == NULL) { @@ -2405,7 +2406,8 @@ Ex_Lob_Error ExLobsOper ( case Lob_Read: if (storage == Lob_External_HDFS_File) //Allocate storage to read the lob external file name from the - //descriptor tables to get the data from. + //descriptor tables to get the data from. + // TODO: do we ever take this code path for Hive files? blackBox = new(lobGlobals->getHeap()) char[MAX_LOB_FILE_NAME_LEN+6]; if (subOperation == Lob_Memory) @@ -2440,12 +2442,21 @@ Ex_Lob_Error ExLobsOper ( case Lob_OpenDataCursorSimple: if (openType == 1) { // preopen - sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId); - preOpenObj = new (lobGlobals->getHeap()) ExLobPreOpen(lobPtr, fn, descNumIn, sourceLen, cursorBytes, waited); + char temp1[30]; // big enough for :%Lx: + sprintf(temp1, ":%Lx:",(long long unsigned int)lobName); + fn = lobPtr->getDataFileName(); + fn += temp1; + fn += cursorId; + preOpenObj = new (lobGlobals->getHeap()) ExLobPreOpen(lobPtr, fn.c_str(), descNumIn, sourceLen, + cursorBytes, waited, lobGlobals->getHeap()); lobGlobals->addToPreOpenList(preOpenObj); } else if (openType == 2) { // must open - sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId); - fileName = fn; + char temp2[30]; // big enough for :%Lx: + sprintf(temp2, ":%Lx:",(long long unsigned int)lobName); + fn = lobPtr->getDataFileName(); + fn += temp2; + fn += cursorId; + fileName = fn.c_str(); err = lobPtr->openDataCursor(fileName, Lob_Cursor_Simple, descNumIn, sourceLen, cursorBytes, waited, lobGlobals, (Int32 *)blackBox); } else err = LOB_SUBOPER_ERROR; @@ -2461,9 +2472,15 @@ Ex_Lob_Error ExLobsOper ( break; case Lob_ReadDataCursorSimple: - sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId); - fileName = fn; - err = lobPtr->readDataCursorSimple(fileName, source, sourceLen, retOperLen, lobGlobals); + { + char temp3[30]; // big enough for :%Lx: + sprintf(temp3, ":%Lx:",(long long unsigned int)lobName); + fn = lobPtr->getDataFileName(); + fn += temp3; + fn += cursorId; + fileName = fn.c_str(); + err = lobPtr->readDataCursorSimple(fileName, source, sourceLen, retOperLen, lobGlobals); + } break; case Lob_CloseFile: @@ -2482,9 +2499,15 @@ Ex_Lob_Error ExLobsOper ( break; case Lob_CloseDataCursorSimple: - sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId); - fileName = fn; - err = lobPtr->closeDataCursorSimple(fileName, lobGlobals); + { + char temp4[30]; // big enough for :%Lx: + sprintf(temp4, ":%Lx:",(long long unsigned int)lobName); + fn = lobPtr->getDataFileName(); + fn += temp4; + fn += cursorId; + fileName = fn.c_str(); + err = lobPtr->closeDataCursorSimple(fileName, lobGlobals); + } break; case Lob_Append: @@ -2684,7 +2707,7 @@ prefetchBufList. It continues doing this until end of range is reached or the buffer limit (128MB) has been reached. */ -Ex_Lob_Error ExLob::readDataCursorSimple(char *file, char *tgt, Int64 tgtSize, +Ex_Lob_Error ExLob::readDataCursorSimple(const char *file, char *tgt, Int64 tgtSize, Int64 &operLen, ExLobGlobals *lobGlobals) { int dataOffset; @@ -2775,7 +2798,7 @@ Ex_Lob_Error ExLob::readDataCursorSimple(char *file, char *tgt, Int64 tgtSize, return LOB_OPER_OK; } -Ex_Lob_Error ExLob::closeDataCursorSimple(char *fileName, ExLobGlobals *lobGlobals) +Ex_Lob_Error ExLob::closeDataCursorSimple(const char *fileName, ExLobGlobals *lobGlobals) { cursor_t *cursor = NULL; Int64 secs = 0; @@ -2924,7 +2947,7 @@ Ex_Lob_Error ExLobGlobals::performRequest(ExLobHdfsRequest *request) traceMessage("locking cursor",cursor,__LINE__); cursor->lock_.unlock(); if (cursor->eol_) { // never reaches here ?? - lobPtr->deleteCursor(cursor->name_, this); + lobPtr->deleteCursor(cursor->name_.c_str(), this); } } processPreOpens(); @@ -3018,7 +3041,7 @@ void ExLobCursor::emptyPrefetchList(ExLobGlobals *lobGlobals) // Seems like this is currently unused. // closeDataCusrorSimple takes care of destroying the cursor.But addign code // similar to closeDataCursorSimple for correctness in case it is used in future -Ex_Lob_Error ExLob::deleteCursor(char *cursorName, ExLobGlobals *lobGlobals) +Ex_Lob_Error ExLob::deleteCursor(const char *cursorName, ExLobGlobals *lobGlobals) { cursor_t *cursor = NULL; @@ -3389,7 +3412,7 @@ Ex_Lob_Error ExLobGlobals::processPreOpens() { ExLob *lobPtr = preOpenObj->lobPtr_; - lobPtr->openDataCursor(preOpenObj->cursorName_, Lob_Cursor_Simple, preOpenObj->range_, + lobPtr->openDataCursor(preOpenObj->cursorName_.data(), Lob_Cursor_Simple, preOpenObj->range_, preOpenObj->bufMaxSize_, preOpenObj->maxBytes_, preOpenObj->waited_, this,0); } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/49498dfd/core/sql/exp/ExpLOBaccess.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h index a8e0ea7..a793487 100644 --- a/core/sql/exp/ExpLOBaccess.h +++ b/core/sql/exp/ExpLOBaccess.h @@ -147,7 +147,7 @@ class ExLobRequest Int64 transId_; SB_Transid_Type transIdBig_; SB_Transseq_Type transStartId_; - char descFileName_[MAX_LOB_FILE_NAME_LEN]; + char descFileName_[MAX_LOB_FILE_NAME_LEN]; // TODO: If we ever use this again, change to use string or NAString char blackBox_[MAX_BLACK_BOX_LEN]; Int64 blackBoxLen_; }; @@ -368,7 +368,7 @@ public: Int64 bufMaxSize_; // max size of buffer Int64 maxBytes_; // bytesLeft to prefetch Int64 prefetch_; // prefetch or not to prefetch - char name_[MAX_LOB_FILE_NAME_LEN]; + string name_; // TODO: change to NAString when ExLobCursor is allocated off of an NAHeap Lng32 currentRange_; // current index of file for multi cursor Lng32 endRange_; // end index of file for multi cursor @@ -406,13 +406,13 @@ class ExLob ExLob(); // default constructor ~ExLob(); // default desctructor - Ex_Lob_Error initialize(char *lobFile, Ex_Lob_Mode mode, char *dir, + Ex_Lob_Error initialize(const char *lobFile, Ex_Lob_Mode mode, char *dir, LobsStorage storage, char *hdfsServer, Int64 hdfsPort, char *lobLocation, int bufferSize = 0, short replication =0, int blocksize=0, Int64 lobMaxSize = 0, ExLobGlobals *lobGlobals = NULL); - Ex_Lob_Error initialize(char *lobFile); + Ex_Lob_Error writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOperation, Int64 &descNumOut, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize,Int64 lobGCLimit, char * handleIn, Int32 handleInLen, char *blackBox, Int32 *blackBoxLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals); Ex_Lob_Error writeLobData(char *source, Int64 sourceLen, LobsSubOper subOperation, @@ -425,12 +425,12 @@ class ExLob Ex_Lob_Error readCursor(char *tgt, Int64 tgtSize, char *handleIn, Int32 handleInLen, Int64 &operLen,Int64 transId); Ex_Lob_Error readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, Int64 &operLen,char *handleIn, Int32 handeLenIn,Int64 transId); Ex_Lob_Error readCursorDataSimple(char *tgt, Int64 tgtSize, cursor_t &cursor, Int64 &operLen); - Ex_Lob_Error readDataCursorSimple(char *fileName, char *tgt, Int64 tgtSize, Int64 &operLen, ExLobGlobals *lobGlobals); + Ex_Lob_Error readDataCursorSimple(const char *fileName, char *tgt, Int64 tgtSize, Int64 &operLen, ExLobGlobals *lobGlobals); bool hasNoOpenCursors() { return lobCursors_.empty(); } Ex_Lob_Error openCursor(char *handleIn, Int32 handleInLen,Int64 transId); - Ex_Lob_Error openDataCursor(char *fileName, LobsCursorType type, Int64 range, + Ex_Lob_Error openDataCursor(const char *fileName, LobsCursorType type, Int64 range, Int64 bytesLeft, Int64 bufMaxSize, Int64 prefetch, ExLobGlobals *lobGlobals, Int32 *hdfsDetailError = NULL); - Ex_Lob_Error deleteCursor(char *cursorName, ExLobGlobals *lobGlobals); + Ex_Lob_Error deleteCursor(const char *cursorName, ExLobGlobals *lobGlobals); Ex_Lob_Error fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOffset, Int64 &outSize,NABoolean &isEOD,Int64 transId); Ex_Lob_Error insertData(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize,char *handleIn,Int32 handleInLen, char *blackBox, Int32 blackBoxLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals); Ex_Lob_Error append(char *data, Int64 size, LobsSubOper so, Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemLen,Int64 lobGCLimit, char *handleIn,Int32 handleInLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals); @@ -445,7 +445,7 @@ class ExLob Ex_Lob_Error closeFile(); LobInputOutputFileType fileType(char *ioFileName); Ex_Lob_Error closeCursor(char *handleIn, Int32 handleInLen); - Ex_Lob_Error closeDataCursorSimple(char *fileName, ExLobGlobals *lobGlobals); + Ex_Lob_Error closeDataCursorSimple(const char *fileName, ExLobGlobals *lobGlobals); Ex_Lob_Error doSanityChecks(char *dir, LobsStorage storage, Int32 handleInLen, Int32 handleOutLen, @@ -458,7 +458,7 @@ class ExLob Ex_Lob_Error lockDesc(); Ex_Lob_Error unlockDesc(); - char *getDataFileName() { return lobDataFile_; } + const char *getDataFileName() { return lobDataFile_.c_str(); } int getErrNo(); @@ -515,7 +515,7 @@ class ExLob public: - char lobDataFile_[MAX_LOB_FILE_NAME_LEN]; + string lobDataFile_; // TODO: change to NAString when ExLobCursor is allocated off of an NAHeap lobCursors_t lobCursors_; ExLobLock lobCursorLock_; LobsStorage storage_; @@ -562,21 +562,24 @@ class ExLobPreOpen { public : - ExLobPreOpen(ExLob *lobPtr, char *cursorName, Int64 range, Int64 bufMaxSize, Int64 maxBytes, Int64 waited) + ExLobPreOpen(ExLob *lobPtr, const char *cursorName, Int64 range, + Int64 bufMaxSize, Int64 maxBytes, Int64 waited, + NAHeap * heap) : lobPtr_(lobPtr), + cursorName_(cursorName,heap), range_(range), bufMaxSize_(bufMaxSize), maxBytes_(maxBytes), waited_(waited) { - strcpy(cursorName_, cursorName); + // nothing else to do } ~ExLobPreOpen(); public : ExLob *lobPtr_; - char cursorName_[MAX_LOB_FILE_NAME_LEN + 16]; + NAString cursorName_; Int64 range_; Int64 bufMaxSize_; Int64 maxBytes_;
