Repository: trafodion Updated Branches: refs/heads/master 81640f930 -> 052f7faa4
Performance changes for LOB and increasing limit for larger varchars/bobs to use with batch input. Project: http://git-wip-us.apache.org/repos/asf/trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/trafodion/commit/528e9c60 Tree: http://git-wip-us.apache.org/repos/asf/trafodion/tree/528e9c60 Diff: http://git-wip-us.apache.org/repos/asf/trafodion/diff/528e9c60 Branch: refs/heads/master Commit: 528e9c60345234e554d32db637c82728b63c945b Parents: efaebe1 Author: Sandhya Sundaresan <[email protected]> Authored: Thu Mar 29 05:56:15 2018 +0000 Committer: Sandhya Sundaresan <[email protected]> Committed: Thu Mar 29 05:56:15 2018 +0000 ---------------------------------------------------------------------- core/sql/common/CharType.cpp | 2 +- core/sql/common/CharType.h | 27 ++++++----- core/sql/exp/ExpLOBaccess.cpp | 27 ++++++----- core/sql/exp/exp_attrs.cpp | 12 ++--- core/sql/exp/exp_attrs.h | 6 ++- core/sql/generator/GenExpGenerator.cpp | 7 +-- core/sql/generator/GenRelExeUtil.cpp | 5 +- core/sql/generator/GenRelJoin.cpp | 8 ++-- core/sql/generator/GenRelPackedRows.cpp | 7 ++- core/sql/generator/Generator.h | 1 + core/sql/optimizer/BindItemExpr.cpp | 12 +++-- core/sql/optimizer/ItemFunc.h | 34 ++++++++------ core/sql/optimizer/ObjectNames.cpp | 2 +- core/sql/optimizer/SynthType.cpp | 8 ++-- core/sql/parser/sqlparser.y | 4 +- core/sql/qmscommon/QRLogger.cpp | 2 +- core/sql/sqlcomp/CmpMain.cpp | 8 +++- core/sql/sqlcomp/nadefaults.cpp | 68 ++++++++++++++++------------ 18 files changed, 139 insertions(+), 101 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/common/CharType.cpp ---------------------------------------------------------------------- diff --git a/core/sql/common/CharType.cpp b/core/sql/common/CharType.cpp index 9b7566b..cc6c18a 100644 --- a/core/sql/common/CharType.cpp +++ b/core/sql/common/CharType.cpp @@ -1477,7 +1477,7 @@ SQLlob::SQLlob(NAMemory *h, ) : NAType(h, (ev == NA_LOB_TYPE ? LiteralLOB : LiteralLOB) , ev - , (externalFormat ? extFormatLen : 512) + , extFormatLen , allowSQLnull , allowSQLnull ? SQL_NULL_HDR_SIZE : 0 , TRUE http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/common/CharType.h ---------------------------------------------------------------------- diff --git a/core/sql/common/CharType.h b/core/sql/common/CharType.h index c92a7db..61eee56 100644 --- a/core/sql/common/CharType.h +++ b/core/sql/common/CharType.h @@ -735,16 +735,21 @@ public: SQLlob(NAMemory *h, NABuiltInTypeEnum ev, - Int64 lobLength, - LobsStorage lobStorage, + Int64 lobLength=1024, + LobsStorage lobStorage=Lob_Invalid_Storage, NABoolean allowSQLnull = TRUE, NABoolean inlineIfPossible = FALSE, NABoolean externalFormat = FALSE, - Lng32 extFormatLen = 100); + Lng32 extFormatLen=1024 ); SQLlob(const SQLlob & aLob,NAMemory * heap) - :NAType(aLob,heap), - charSet_(CharInfo::UnknownCharSet) - {} + :NAType(aLob,heap) + { + lobLength_ = aLob.lobLength_; + lobStorage_ = aLob.lobStorage_; + externalFormat_ = aLob.externalFormat_; + extFormatLen_ = aLob.extFormatLen_; + setCharSet(CharInfo::ISO88591);//lobhandle can only be in ISO format + } //is this type a blob/clob virtual NABoolean isLob() const {return TRUE;}; @@ -798,14 +803,14 @@ class SQLBlob : public SQLlob public: SQLBlob(NAMemory *h, - Int64 blobLength, + Int64 blobLength=1024, LobsStorage lobStorage = Lob_Invalid_Storage, NABoolean allowSQLnull = TRUE, NABoolean inlineIfPossible = FALSE, NABoolean externalFormat = FALSE, Lng32 extFormatLen = 1024); SQLBlob(const SQLBlob & aBlob,NAMemory * heap) - :SQLlob(aBlob,heap) + :SQLlob(aBlob,heap) {} virtual short getFSDatatype() const {return REC_BLOB;} @@ -838,14 +843,14 @@ class SQLClob : public SQLlob public: SQLClob(NAMemory *h, - Int64 blobLength, + Int64 blobLength=1024, LobsStorage lobStorage = Lob_Invalid_Storage, NABoolean allowSQLnull = TRUE, NABoolean inlineIfPossible = FALSE, NABoolean externalFormat = FALSE, Lng32 extFormatLen = 1024); SQLClob(const SQLClob & aClob,NAMemory * heap) - :SQLlob(aClob,heap) + :SQLlob(aClob,heap) {} virtual short getFSDatatype() const {return REC_CLOB;} @@ -864,6 +869,6 @@ public: private: }; // class SQLClob -// sss #endif + #endif /* CHARTYPE_H */ http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/exp/ExpLOBaccess.cpp ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp index b5a427b..93b340f 100644 --- a/core/sql/exp/ExpLOBaccess.cpp +++ b/core/sql/exp/ExpLOBaccess.cpp @@ -304,15 +304,14 @@ Ex_Lob_Error ExLob::writeData(Int64 offset, char *data, Int32 size, Int64 &operL if (fInfo == NULL) { return LOB_DATA_FILE_NOT_FOUND_ERROR; } + if (fdData_) + { + hdfsCloseFile(fs_, fdData_); + fdData_=NULL; + } + openFlags_ = O_WRONLY | O_APPEND; + fdData_ = hdfsOpenFile(fs_, lobDataFile_.data(), openFlags_, 0, 0, 0); } - hdfsCloseFile(fs_, fdData_); - fdData_=NULL; - openFlags_ = O_WRONLY | O_APPEND; - fdData_ = hdfsOpenFile(fs_, lobDataFile_.data(), openFlags_, 0, 0, 0); - if (!fdData_) { - openFlags_ = -1; - return LOB_DATA_FILE_OPEN_ERROR; - } if ((operLen = hdfsWrite(fs_, fdData_, data, size)) == -1) { return LOB_DATA_WRITE_ERROR; @@ -946,6 +945,10 @@ Ex_Lob_Error ExLob::writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOpe LobInputOutputFileType srcFileType = fileType(source); if (srcFileType != HDFS_FILE) return LOB_SOURCE_FILE_READ_ERROR; + //Check if external file exists + Int64 sourceEOD = 0; + if (statSourceFile(source, sourceEOD) != LOB_OPER_OK) + return LOB_SOURCE_FILE_READ_ERROR; } // Calculate sourceLen for each subOper. if ((subOper == Lob_File)) @@ -1256,6 +1259,8 @@ Ex_Lob_Error ExLob::insertSelect(ExLob *srcLobPtr, // we have received the external data file name from the descriptor table // replace the contents of the lobDataFile with this name str_cpy_all(extFileName, blackBox, blackBoxLen); + extFileName[blackBoxLen]= '\0'; + extFileNameLen = blackBoxLen; // Now insert this into the target lob descriptor @@ -1822,8 +1827,8 @@ Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset, hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.data()); if (fInfo) dataOffset = fInfo->mSize; - - if ((lobGCLimit != 0) && (dataOffset > lobGCLimit)) // 5 GB default + // if -1, don't do GC or if reached the limit do GC + if ((lobGCLimit != -1) && (dataOffset > lobGCLimit)) { str_sprintf(logBuf,"Starting GC. Current Offset : %ld",dataOffset); lobDebugInfo(logBuf,0,__LINE__,lobTrace_); @@ -1876,7 +1881,7 @@ Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int Ex_Lob_Error rc = LOB_OPER_OK; char logBuf[4096]; lobDebugInfo("In ExLob::compactLobDataFile",0,__LINE__,lobTrace_); - Int64 maxMemChunk = 1024*1024*1024; //1GB limit for intermediate buffer for transfering data + Int64 maxMemChunk = 100*1024*1024; //100 MB limit for intermediate buffer for transfering data // make some temporary file names size_t len = lobDataFile_.length(); http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/exp/exp_attrs.cpp ---------------------------------------------------------------------- diff --git a/core/sql/exp/exp_attrs.cpp b/core/sql/exp/exp_attrs.cpp index 8ded216..7215168 100644 --- a/core/sql/exp/exp_attrs.cpp +++ b/core/sql/exp/exp_attrs.cpp @@ -316,15 +316,13 @@ void Attributes::displayContents(Space * space, Int32 operandNum, if ((getDatatype() == REC_BLOB) || (getDatatype() == REC_CLOB)) { - Int16 precision = getPrecision(); - UInt16 scale = getScaleAsUI(); + - Lng32 lobLen = (precision << 16); - lobLen += scale; - - Int64 ll = (Int64)lobLen; + Int64 ll = getLength(); + if (isLengthInKB()) + ll = ll*1024; // Int64 ll = (Int64)getPrecision() * 1000 + (Int64)getScale(); - str_sprintf(buf, " LobLength = %ld Mb", ll); + str_sprintf(buf, " LobLength = %ld bytes", ll); space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sizeof(short)); } http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/exp/exp_attrs.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/exp_attrs.h b/core/sql/exp/exp_attrs.h index 32a818d..62fe617 100644 --- a/core/sql/exp/exp_attrs.h +++ b/core/sql/exp/exp_attrs.h @@ -378,6 +378,9 @@ public: NABoolean isForceFixed() { return (flags_ & FORCE_FIXED_) != 0; } void setForceFixed() { flags_ |= FORCE_FIXED_; } + NABoolean isLengthInKB() { return (flags_ & LENGTH_IN_KB_) != 0; } + void setLengthInKB() { flags_ |= LENGTH_IN_KB_; } + // Bulk move flags void setBulkMoveable( NABoolean flag = TRUE ) { (flag ? flags_ |= BULK_MOVE_ : flags_ &= ~BULK_MOVE_); } NABoolean isBulkMoveable() { return (flags_ & BULK_MOVE_) != 0; } @@ -584,9 +587,10 @@ private: CASEINSENSITIVE_ = 0x0400, // caseinsensitive char/varchar datatype - FORCE_FIXED_ = 0x0800 // Force this attribute to be treated as fixed + FORCE_FIXED_ = 0x0800, // Force this attribute to be treated as fixed // in an aligned row. Used by HashGroupby for // varchar aggregates + LENGTH_IN_KB_ =0x1000 // Indicates length is in KB }; http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/generator/GenExpGenerator.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenExpGenerator.cpp b/core/sql/generator/GenExpGenerator.cpp index cde3481..92b2aa4 100644 --- a/core/sql/generator/GenExpGenerator.cpp +++ b/core/sql/generator/GenExpGenerator.cpp @@ -359,13 +359,8 @@ Attributes * ExpGenerator::convertNATypeToAttributes else if (naType->getTypeQualifier() == NA_LOB_TYPE) { + SQLlob *lobType = (SQLlob *) naType; - - Lng32 lobLen = (Lng32)lobType->getLobLength(); - Int16 precision = (lobLen & 0xFFFF0000) >> 16; - Int16 scale = lobLen & 0xFFFF; - attr->setPrecision(precision); - attr->setScale(scale); attr->setCharSet(lobType->getCharSet()); } http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/generator/GenRelExeUtil.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelExeUtil.cpp b/core/sql/generator/GenRelExeUtil.cpp index c667f99..5df0849 100644 --- a/core/sql/generator/GenRelExeUtil.cpp +++ b/core/sql/generator/GenRelExeUtil.cpp @@ -4410,7 +4410,10 @@ short ExeUtilLobUpdate::codeGen(Generator * generator) exe_util_lobupdate_tdb->setTotalBufSize(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE)*1024*1024); exe_util_lobupdate_tdb->setLobMaxSize( CmpCommon::getDefaultNumeric(LOB_MAX_SIZE) * 1024 * 1024); exe_util_lobupdate_tdb->setLobMaxChunkSize(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE)*1024*1024); - exe_util_lobupdate_tdb->setLobGCLimit(CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE)*1024*1024); + if ((CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE) >=0)) + exe_util_lobupdate_tdb->setLobGCLimit(CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE)*1024*1024); + else + exe_util_lobupdate_tdb->setLobGCLimit(CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE)); generator->setCriDesc(givenDesc, Generator::DOWN); http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/generator/GenRelJoin.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelJoin.cpp b/core/sql/generator/GenRelJoin.cpp index cc97b1d..f54a48a 100644 --- a/core/sql/generator/GenRelJoin.cpp +++ b/core/sql/generator/GenRelJoin.cpp @@ -3482,9 +3482,11 @@ short NestedJoin::codeGen(Generator * generator) // Make sure that the LHS up queue can grow as large as the RHS down // queue. - if(tdb1->getMaxQueueSizeUp() < tdb2->getInitialQueueSizeDown()) { - tdb1->setMaxQueueSizeUp(tdb2->getInitialQueueSizeDown()); - } + //We can't let upqueue of unpack to grow beyond what was calculated and set earlier in codeGen of the unpack operator + if (tdb1->getClassID() != ComTdb::ex_UNPACKROWS) + if(tdb1->getMaxQueueSizeUp() < tdb2->getInitialQueueSizeDown()) { + tdb1->setMaxQueueSizeUp(tdb2->getInitialQueueSizeDown()); + } // If this NestedJoin itself is not on the RHS of a Flow/NestedJoin, // Then reset the largeQueueSize to 0. http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/generator/GenRelPackedRows.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelPackedRows.cpp b/core/sql/generator/GenRelPackedRows.cpp index adfc388..cc3e6fc 100644 --- a/core/sql/generator/GenRelPackedRows.cpp +++ b/core/sql/generator/GenRelPackedRows.cpp @@ -426,10 +426,11 @@ PhysUnPackRows::codeGen(Generator *generator) ULng32 rowsetSize = getGroupAttr()->getOutputLogPropList()[0]->getResultCardinality().value(); double memoryLimitPerInstance = - ActiveSchemaDB()->getDefaults().getAsLong(EXE_MEMORY_FOR_UNPACK_ROWS_IN_MB) * 1024 * 1024; + (ActiveSchemaDB()->getDefaults().getAsLong(EXE_MEMORY_FOR_UNPACK_ROWS_IN_MB) * 1024 * 1024)/2; // At runtime (ExUnpackRowsTcb::ExUnpackRowsTcb) we allocate twice the size of the queue. So ensure it fits in to half the specified limit. rowsetSize = (rowsetSize < 1024 ? 1024 : rowsetSize); - double estimatedMemory = rowsetSize * unPackColsTupleLen; + + double estimatedMemory = (Int64)rowsetSize * (Int64)unPackColsTupleLen; if (estimatedMemory > memoryLimitPerInstance) { @@ -466,6 +467,8 @@ PhysUnPackRows::codeGen(Generator *generator) // generator->removeAll(localMapTable); + //Turn off override of queue sizes for unpack. Once set they should remain. + generator->setMakeOnljLeftQueuesBig(FALSE); // Add the explain Information for this node to the EXPLAIN // Fragment. Set the explainTuple pointer in the generator so // the parent of this node can get a handle on this explainTuple. http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/generator/Generator.h ---------------------------------------------------------------------- diff --git a/core/sql/generator/Generator.h b/core/sql/generator/Generator.h index e4524a8..8f2f50d 100644 --- a/core/sql/generator/Generator.h +++ b/core/sql/generator/Generator.h @@ -1580,6 +1580,7 @@ public: // Helpers for the special ONLJ queue sizing defaults. NABoolean const getMakeOnljLeftQueuesBig() { return makeOnljLeftQueuesBig_; } + void setMakeOnljLeftQueuesBig(NABoolean x) {makeOnljLeftQueuesBig_ = x;} ULng32 const getOnljLeftUpQueue() { return onljLeftUpQueue_; } ULng32 const getOnljLeftDownQueue() {return onljLeftDownQueue_; } http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/optimizer/BindItemExpr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/BindItemExpr.cpp b/core/sql/optimizer/BindItemExpr.cpp index 3b71061..282c852 100644 --- a/core/sql/optimizer/BindItemExpr.cpp +++ b/core/sql/optimizer/BindItemExpr.cpp @@ -6103,20 +6103,22 @@ ItemExpr *Assign::bindNode(BindWA *bindWA) NAType * newType = NULL; - double lob_input_limit_for_batch = CmpCommon::getDefaultNumeric(LOB_INPUT_LIMIT_FOR_BATCH); + double lob_input_limit_for_batch = CmpCommon::getDefaultNumeric(LOB_INPUT_LIMIT_FOR_BATCH)*1024; double lob_size = lobType.getLobLength(); if (fs_datatype == REC_CLOB) { - newType = new (bindWA->wHeap()) SQLClob(bindWA->wHeap(), (CmpCommon::getDefaultNumeric(LOB_MAX_SIZE) * 1024 * 1024), + newType = new (bindWA->wHeap()) SQLClob(bindWA->wHeap(), lob_input_limit_for_batch < lob_size ? lob_input_limit_for_batch : lob_size, lobType.getLobStorage(), - TRUE, FALSE, TRUE, + TRUE, FALSE, FALSE, lob_input_limit_for_batch < lob_size ? lob_input_limit_for_batch : lob_size); } else { - newType = new (bindWA->wHeap()) SQLBlob(bindWA->wHeap(), (CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024), + newType = new (bindWA->wHeap()) SQLBlob(bindWA->wHeap(),lob_input_limit_for_batch < lob_size ? lob_input_limit_for_batch : lob_size , lobType.getLobStorage(), - TRUE, FALSE, TRUE, + TRUE, FALSE, FALSE, lob_input_limit_for_batch < lob_size ? lob_input_limit_for_batch : lob_size); } + + CMPASSERT(lob_input_limit_for_batch < INT_MAX); vid1.coerceType(*newType, NA_LOB_TYPE); if (bindWA->getCurrentScope()->context()->inUpdate()) { http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/optimizer/ItemFunc.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/ItemFunc.h b/core/sql/optimizer/ItemFunc.h index 0435521..ca16965 100644 --- a/core/sql/optimizer/ItemFunc.h +++ b/core/sql/optimizer/ItemFunc.h @@ -2869,9 +2869,9 @@ public: obj_(obj), lobNum_(-1), lobStorageType_(Lob_Invalid_Storage), - lobMaxSize_((Int64)CmpCommon::getDefaultNumeric(LOB_MAX_SIZE) * 1024 * 1024), - lobMaxChunkMemSize_(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE)), - lobGCLimit_(CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE)), + lobMaxSize_((Int64)CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024), + lobMaxChunkMemSize_((Int64)CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE)*1024*1024), + lobGCLimit_((Int64) CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE)*1024*1024), hdfsPort_((Lng32)CmpCommon::getDefaultNumeric(LOB_HDFS_PORT)), hdfsServer_( CmpCommon::getDefaultString(LOB_HDFS_SERVER)) { @@ -2911,8 +2911,14 @@ public: LobsStorage &lobStorageType() { return lobStorageType_; } NAString &lobStorageLocation() { return lobStorageLocation_; } Int64 getLobMaxSize() {return lobMaxSize_; } - Int64 getLobMaxChunkMemSize() { return lobMaxChunkMemSize_*1024*1024;} - Int64 getLobGCLimit() { return lobGCLimit_*1024*1024;} + Int64 getLobMaxChunkMemSize() { return lobMaxChunkMemSize_;} + Int64 getLobGCLimit() + { + if (lobGCLimit_>0) + return (Int64)lobGCLimit_; + else + return -1; + } Int32 getLobHdfsPort() { return hdfsPort_;} NAString &getLobHdfsServer(){return hdfsServer_;} protected: @@ -2922,8 +2928,8 @@ public: LobsStorage lobStorageType_; NAString lobStorageLocation_; Int64 lobMaxSize_; // In byte units - Int32 lobMaxChunkMemSize_; //In MB Units - Int32 lobGCLimit_ ;//In MB Units + Int64 lobMaxChunkMemSize_; //In MB Units + Int64 lobGCLimit_ ;//In MB Units Int32 hdfsPort_; NAString hdfsServer_; @@ -2943,7 +2949,7 @@ class LOBinsert : public LOBoper objectUID_(-1), append_(isAppend), lobAsVarchar_(treatLobAsVarchar), - lobSize_(0), + lobSize_((Int64)CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024), fsType_(REC_BLOB) {}; @@ -2968,7 +2974,7 @@ class LOBinsert : public LOBoper // Lng32 & lobNum() { return lobNum_; } - Lng32 & lobSize() { return lobSize_; } + Int64 & lobSize() { return lobSize_; } Lng32 & lobFsType() { return fsType_; } NABoolean lobAsVarchar() const { return lobAsVarchar_;} @@ -2984,7 +2990,7 @@ class LOBinsert : public LOBoper // column this blob is being inserted into. // Lng32 lobNum_; - Lng32 lobSize_; + Int64 lobSize_; Lng32 fsType_; @@ -3048,7 +3054,7 @@ class LOBupdate : public LOBoper NABoolean treatLobAsVarchar =FALSE) : LOBoper(ITM_LOBUPDATE, val1Ptr, val2Ptr,val3Ptr,fromObj), objectUID_(-1), - lobSize_(0), + lobSize_((Int64)CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024), append_(isAppend), lobAsVarchar_(treatLobAsVarchar) {}; @@ -3070,7 +3076,7 @@ class LOBupdate : public LOBoper Int64 & updatedTableObjectUID() { return objectUID_; } NAString &updatedTableSchemaName() { return schName_; } - Lng32 & lobSize() { return lobSize_; } + Int64 & lobSize() { return lobSize_; } NABoolean lobAsVarchar() const { return lobAsVarchar_;} private: // ---------------------------------------------------------------// @@ -3083,7 +3089,7 @@ class LOBupdate : public LOBoper NABoolean append_; - Lng32 lobSize_; + Int64 lobSize_; NABoolean lobAsVarchar_;//This means this lob insert will get it's input data // in varchar format }; // class LOBupdate @@ -3092,7 +3098,7 @@ class LOBconvert : public LOBoper { public: - LOBconvert(ItemExpr *val1Ptr, ObjectType toObj, Lng32 tgtSize = 32000) + LOBconvert(ItemExpr *val1Ptr, ObjectType toObj, Lng32 tgtSize= CmpCommon::getDefaultNumeric(LOB_OUTPUT_SIZE) ) : LOBoper(ITM_LOBCONVERT, val1Ptr, NULL,NULL,toObj), tgtSize_(tgtSize) {}; http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/optimizer/ObjectNames.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/ObjectNames.cpp b/core/sql/optimizer/ObjectNames.cpp index fb4f9b1..6547a98 100644 --- a/core/sql/optimizer/ObjectNames.cpp +++ b/core/sql/optimizer/ObjectNames.cpp @@ -710,7 +710,7 @@ NABoolean QualifiedName::isHbaseCellOrRow() const NABoolean QualifiedName::isLOBDesc() const { - if (getObjectName().index(LOB_DESC_HANDLE_PREFIX) == 0) + if ((getObjectName().index(LOB_DESC_HANDLE_PREFIX) == 0) || (getObjectName().index(LOB_DESC_CHUNK_PREFIX) ==0)) return TRUE; else http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/optimizer/SynthType.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/SynthType.cpp b/core/sql/optimizer/SynthType.cpp index 5494472..72d15e6 100644 --- a/core/sql/optimizer/SynthType.cpp +++ b/core/sql/optimizer/SynthType.cpp @@ -6680,7 +6680,7 @@ const NAType *LOBoper::synthesizeType() { // Return blob or clob type - NAType *result = new HEAP SQLBlob(HEAP, 1000); + NAType *result = new HEAP SQLBlob(HEAP, ((Int64) CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024)); if (child(0)) { @@ -6689,12 +6689,12 @@ const NAType *LOBoper::synthesizeType() if (typ1.getFSDatatype() == REC_BLOB) { - result = new HEAP SQLBlob(HEAP, 1000, Lob_Local_File, + result = new HEAP SQLBlob(HEAP, ((Int64) CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024), Lob_Invalid_Storage, typ1.supportsSQLnull()); } else if (typ1.getFSDatatype() == REC_CLOB) { - result = new HEAP SQLClob(HEAP, 1000, Lob_Invalid_Storage, + result = new HEAP SQLClob(HEAP, ((Int64) CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024), Lob_Invalid_Storage, typ1.supportsSQLnull()); } } @@ -6923,7 +6923,7 @@ const NAType *LOBconvertHandle::synthesizeType() return NULL; } - result = new HEAP SQLBlob(HEAP, 1000, Lob_Invalid_Storage, typ1.supportsSQLnull(), FALSE, + result = new HEAP SQLBlob(HEAP, ((Int64) CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024), Lob_Invalid_Storage, typ1.supportsSQLnull(), FALSE, FALSE); return result; } http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/parser/sqlparser.y ---------------------------------------------------------------------- diff --git a/core/sql/parser/sqlparser.y b/core/sql/parser/sqlparser.y index f4f6da6..edbca46 100755 --- a/core/sql/parser/sqlparser.y +++ b/core/sql/parser/sqlparser.y @@ -11647,7 +11647,7 @@ blob_optional_left_len_right: '(' NUMERIC_LITERAL_EXACT_NO_SCALE optional_lob_un if (CmpCommon::getDefault(TRAF_BLOB_AS_VARCHAR) == DF_ON) { - $$ = (Int64)100000; + $$ = (Int64)CmpCommon::getDefault(TRAF_MAX_CHARACTER_COL_LENGTH ); } else { @@ -11683,7 +11683,7 @@ clob_optional_left_len_right: '(' NUMERIC_LITERAL_EXACT_NO_SCALE optional_lob_un if (CmpCommon::getDefault(TRAF_CLOB_AS_VARCHAR) == DF_ON) { - $$ = (Int64)100000; + $$ = (Int64)CmpCommon::getDefault(TRAF_MAX_CHARACTER_COL_LENGTH ); } else { http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/qmscommon/QRLogger.cpp ---------------------------------------------------------------------- diff --git a/core/sql/qmscommon/QRLogger.cpp b/core/sql/qmscommon/QRLogger.cpp index f4fb75d..b71f190 100644 --- a/core/sql/qmscommon/QRLogger.cpp +++ b/core/sql/qmscommon/QRLogger.cpp @@ -266,7 +266,7 @@ const char* QRLogger::getMyProcessInfo() XPROCESSHANDLE_DECOMPOSE_(&myphandle, &mycpu, &mypin, &mynodenumber,NULL,100, NULL, myprocname,100, &myproclength); myprocname[myproclength] = '\0'; - snprintf(procInfo, 300, "Node Number: %d, CPU: %d, PIN: %d, Process Name: %s", mynodenumber,mycpu, mypin, myprocname); + snprintf(procInfo, 300, "Node Number: %d, CPU: %d, PIN: %d, Process Name: %s", mycpu,mycpu, mypin, myprocname); processInfo_ = procInfo; return processInfo_.data(); http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/sqlcomp/CmpMain.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpMain.cpp b/core/sql/sqlcomp/CmpMain.cpp index b7b8851..650d181 100644 --- a/core/sql/sqlcomp/CmpMain.cpp +++ b/core/sql/sqlcomp/CmpMain.cpp @@ -725,9 +725,13 @@ CmpMain::ReturnStatus CmpMain::sqlcomp(QueryText& input, //IN //if using special tables e.g. using index as base table //select * from table (index_table T018ibc); - //then refresh metadata cache + //then refresh metadata cache. Make an exception for internal exeutil + // statements that use this parserflag. It causes too many long compiles + // and affects performance - for eg LOB access which uses ghost tables and + // has the SPECIALTABLETYPE flag set.. if(Get_SqlParser_Flags(ALLOW_SPECIALTABLETYPE) && - CmpCommon::context()->schemaDB_->getNATableDB()->cachingMetaData()) + CmpCommon::context()->schemaDB_->getNATableDB()->cachingMetaData() && + !Get_SqlParser_Flags(INTERNAL_QUERY_FROM_EXEUTIL) ) CmpCommon::context()->schemaDB_->getNATableDB()->refreshCacheInThisStatement(); MonitorMemoryUsage_Enter("Parser"); http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/sqlcomp/nadefaults.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp index a75c20b..f8f8e75 100644 --- a/core/sql/sqlcomp/nadefaults.cpp +++ b/core/sql/sqlcomp/nadefaults.cpp @@ -1154,7 +1154,7 @@ SDDkwd__(EXE_DIAGNOSTIC_EVENTS, "OFF"), SDDui___(EXE_MEMORY_FOR_PROBE_CACHE_IN_MB, "100"), - SDDui___(EXE_MEMORY_FOR_UNPACK_ROWS_IN_MB, "100"), + SDDui___(EXE_MEMORY_FOR_UNPACK_ROWS_IN_MB, "1024"), // lower-bound memory limit for BMOs/nbmos (in MB) DDui___(EXE_MEMORY_LIMIT_LOWER_BOUND_EXCHANGE, "10"), @@ -1741,21 +1741,21 @@ SDDkwd__(ISO_MAPPING, (char *)SQLCHARSETSTRING_ISO88591), // precision but degraded performance. SDDkwd__(LIMIT_MAX_NUMERIC_PRECISION, "SYSTEM"), - // Size in bytes used to perform garbage collection to lob data file - // default size is 5GB . Change to adjust disk usage. If 0 it means - // don't do GC - DDint__(LOB_GC_LIMIT_SIZE, "5000"), + // Size in MB used to perform garbage collection to lob data file + // Recommended size is 20GB . Change to adjust disk usage. If -1 it means + // don't do GC during insert/update operations. + DDint__(LOB_GC_LIMIT_SIZE, "-1"), DDint__(LOB_HDFS_PORT, "0"), DD_____(LOB_HDFS_SERVER, "default"), - - DDint__(LOB_INPUT_LIMIT_FOR_BATCH, "4096"), + // For JDBC/ODBC batch operations, LOB size limited to 4K bytes + DDint__(LOB_INPUT_LIMIT_FOR_BATCH, "16384"), // Size of memoryin Megabytes used to perform I/O to lob data file - // default size is 512MB . Change to adjust memory usage. - DDint__(LOB_MAX_CHUNK_MEM_SIZE, "512"), - // default size is 10 G (10000 M) - DDint__(LOB_MAX_SIZE, "10000"), - // (unused)default size is 32000. Change this to extract more data into memory. + // default size is 128MB . Change to adjust memory usage. + DDint__(LOB_MAX_CHUNK_MEM_SIZE, "128"), + // default size is 5 G (5120 M) + DDint__(LOB_MAX_SIZE, "5120"), + // default size is 32000bytes. Change this to extract more data into memory. DDui___(LOB_OUTPUT_SIZE, "32000"), DD_____(LOB_STORAGE_FILE_DIR, "/user/trafodion/lobs"), @@ -1915,7 +1915,7 @@ SDDkwd__(ISO_MAPPING, (char *)SQLCHARSETSTRING_ISO88591), DDui___(MEMORY_LIMIT_QCACHE_UPPER_KB, "0"), // Checked at compile time. Set to -1 to disable check. // Value should be >= EXE_MEMORY_FOR_UNPACK_ROWS_IN_MB - DDint__(MEMORY_LIMIT_ROWSET_IN_MB, "500"), + DDint__(MEMORY_LIMIT_ROWSET_IN_MB, "1024"), // SQL/MX Compiler/Optimzer Memory Monitor. DDkwd__(MEMORY_MONITOR, "OFF"), @@ -5427,7 +5427,32 @@ enum DefaultConstants NADefaults::validateAndInsert(const char *attrName, } } break; + case LOB_GC_LIMIT_SIZE: + if ((atoi(value.data()) < -1) || (((atoi(value.data()) >0) && (atoi(value.data())) <= 512))) + { + *CmpCommon::diags() << DgSqlCode(-2055) + << DgString0(value) + << DgString1(lookupAttrName(attrEnum)); + } + break; + case LOB_MAX_CHUNK_MEM_SIZE: + if (atoi(value.data()) < 0 || atoi(value.data()) > 5120) + { + *CmpCommon::diags() << DgSqlCode(-2055) + << DgString0(value) + << DgString1(lookupAttrName(attrEnum)); + } + break; + + case LOB_INPUT_LIMIT_FOR_BATCH: + if (atoi(value.data()) <0 && (atoi(value.data())*1024) > INT_MAX ) + { + *CmpCommon::diags() << DgSqlCode(-2055) + << DgString0(value) + << DgString1(lookupAttrName(attrEnum)); + } + break; default: break; } @@ -6466,20 +6491,6 @@ DefaultToken NADefaults::token(Int32 attrEnum, isValid = TRUE; break; - case LOB_OUTPUT_SIZE: - if (tok >=0 && tok <= 512000) - isValid = TRUE; - break; - - case LOB_MAX_CHUNK_MEM_SIZE: - if (tok >=0 && tok <= 512000) - isValid = TRUE; - break; - - case LOB_GC_LIMIT_SIZE: - if (tok >= 0 ) - isValid=TRUE; - case TRAF_TRANS_TYPE: if (tok == DF_MVCC || tok == DF_SSCC) isValid = TRUE; @@ -6782,8 +6793,7 @@ void NADefaults::setSchemaAsLdapUser(const NAString val) insert(SCHEMA, schName); } else - { - *CmpCommon::diags() << DgSqlCode(-2055) + { *CmpCommon::diags() << DgSqlCode(-2055) << DgString0(schName) << DgString1("SCHEMA"); }
