Repository: incubator-trafodion Updated Branches: refs/heads/master f7cf2b86b -> d2c464c8d
[TRAFODION-1700] Upsert with omitted default value columns leave the aligned format table in corrupted state. [TRAFODION-1847] Upsert with omitted timestamp columns having current_timestamp as default in a non-aligned format table returns wrong value for this column For TRAFODION-1847, the upsert is transformed into merge. For TRAFODION-1700, the upsert is transformed into merge when the CQD TRAF_UPSERT_WITH_INSERT_DEFAULT_SEMANTICS is set to OFF. By default this CQD is set to OFF. When this CQD is ON, the upsert will add a new row with the omitted columns filled with default values always. Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/a8210ea0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/a8210ea0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/a8210ea0 Branch: refs/heads/master Commit: a8210ea0fb73d63b244003d945fdcd00617346cd Parents: f712ccb Author: selvaganesang <[email protected]> Authored: Thu Feb 25 18:12:44 2016 +0000 Committer: selvaganesang <[email protected]> Committed: Thu Feb 25 18:12:44 2016 +0000 ---------------------------------------------------------------------- core/sql/comexe/ComTdbHbaseAccess.cpp | 3 + core/sql/comexe/ComTdbHbaseAccess.h | 4 +- core/sql/executor/ExHbaseAccess.cpp | 22 +++++- core/sql/executor/ExHbaseAccess.h | 1 + core/sql/executor/ExHbaseIUD.cpp | 11 ++- core/sql/generator/GenPreCode.cpp | 13 ---- core/sql/generator/GenRelUpdate.cpp | 112 +++++++++++++++-------------- core/sql/optimizer/BindRelExpr.cpp | 83 ++++++++++++++++----- core/sql/optimizer/ItemOther.h | 7 ++ core/sql/optimizer/RelExpr.cpp | 2 +- core/sql/optimizer/RelUpdate.h | 10 +-- core/sql/sqlcomp/DefaultConstants.h | 3 + core/sql/sqlcomp/nadefaults.cpp | 1 + 13 files changed, 178 insertions(+), 94 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a8210ea0/core/sql/comexe/ComTdbHbaseAccess.cpp ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbHbaseAccess.cpp b/core/sql/comexe/ComTdbHbaseAccess.cpp index 8276553..7d3c3b1 100644 --- a/core/sql/comexe/ComTdbHbaseAccess.cpp +++ b/core/sql/comexe/ComTdbHbaseAccess.cpp @@ -282,6 +282,7 @@ ComTdbHbaseAccess::ComTdbHbaseAccess( listOfUpDeldColNames_(NULL), listOfMergedColNames_(NULL), listOfIndexesAndTable_(NULL), + listOfOmittedColNames_(NULL), keyInfo_(NULL), keyColName_(NULL), @@ -438,6 +439,7 @@ Long ComTdbHbaseAccess::pack(void * space) listOfFetchedColNames_.pack(space); listOfUpDeldColNames_.pack(space); listOfMergedColNames_.pack(space); + listOfOmittedColNames_.pack(space); listOfIndexesAndTable_.pack(space); keyInfo_.pack(space); keyColName_.pack(space); @@ -508,6 +510,7 @@ Lng32 ComTdbHbaseAccess::unpack(void * base, void * reallocator) if(listOfFetchedColNames_.unpack(base, reallocator)) return -1; if(listOfUpDeldColNames_.unpack(base, reallocator)) return -1; if(listOfMergedColNames_.unpack(base, reallocator)) return -1; + if(listOfOmittedColNames_.unpack(base, reallocator)) return -1; if(listOfIndexesAndTable_.unpack(base, reallocator)) return -1; if(keyInfo_.unpack(base, reallocator)) return -1; if(keyColName_.unpack(base)) return -1; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a8210ea0/core/sql/comexe/ComTdbHbaseAccess.h ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbHbaseAccess.h b/core/sql/comexe/ComTdbHbaseAccess.h index c3b11f6..909a2ed 100644 --- a/core/sql/comexe/ComTdbHbaseAccess.h +++ b/core/sql/comexe/ComTdbHbaseAccess.h @@ -610,8 +610,9 @@ public: Queue* listOfDeletedColNames() { return listOfUpDeldColNames_; } Queue* listOfMergedColNames() { return listOfMergedColNames_; } Queue* listOfIndexesAndTable() { return listOfIndexesAndTable_; } - + Queue* listOfOmittedColNames() { return listOfOmittedColNames_; } void setListOfIndexesAndTable(Queue* val) {listOfIndexesAndTable_ = val; } + void setListOfOmittedColNames(Queue* val) {listOfOmittedColNames_ = val; } // overloading listOfUpdatedColNames and listOfMergedColNames...for now. Queue* listOfHbaseFilterColNames() { return listOfUpDeldColNames_; } @@ -977,6 +978,7 @@ public: QueuePtr listOfUpDeldColNames_; QueuePtr listOfMergedColNames_; QueuePtr listOfIndexesAndTable_; // used by bulk load + QueuePtr listOfOmittedColNames_; // information about key ranges keyRangeGenPtr keyInfo_; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a8210ea0/core/sql/executor/ExHbaseAccess.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExHbaseAccess.cpp b/core/sql/executor/ExHbaseAccess.cpp index 188d270..3072b6c 100644 --- a/core/sql/executor/ExHbaseAccess.cpp +++ b/core/sql/executor/ExHbaseAccess.cpp @@ -2510,6 +2510,7 @@ short ExHbaseAccessTcb::copyRowIDToDirectBuffer(HbaseStr &rowID) short ExHbaseAccessTcb::createDirectRowBuffer( UInt16 tuppIndex, char * tuppRow, Queue * listOfColNames, + Queue * listOfOmittedColNames, NABoolean isUpdate, std::vector<UInt32> * posVec, double samplingRate ) @@ -2548,7 +2549,8 @@ short ExHbaseAccessTcb::createDirectRowBuffer( UInt16 tuppIndex, Attributes * attr; int numCols = 0; short *numColsPtr; - + char *str_1; + NABoolean omittedColFound; allocateDirectRowBufferForJNI(rowTD->numAttrs()); BYTE *rowCurPtr = (BYTE *)row_.val; @@ -2556,9 +2558,12 @@ short ExHbaseAccessTcb::createDirectRowBuffer( UInt16 tuppIndex, row_.len += sizeof(short); rowCurPtr += sizeof(short); listOfColNames->position(); + for (Lng32 i = 0; i < rowTD->numAttrs(); i++) { + Attributes * attr; + if (!posVec) attr = rowTD->getAttr(i); else @@ -2572,6 +2577,21 @@ short ExHbaseAccessTcb::createDirectRowBuffer( UInt16 tuppIndex, { extractColNameFields((char*)listOfColNames->getCurr(), colNameLen, colName); + if (listOfOmittedColNames != NULL) { + omittedColFound = FALSE; + listOfOmittedColNames->position(); + while ((str_1 = (char *)listOfOmittedColNames->getNext()) != NULL) { + str = (char*)listOfColNames->getCurr(); + if (memcmp(str, str_1, colNameLen+2) == 0) { + omittedColFound = TRUE; + break; + } + } + if (omittedColFound) { + listOfColNames->advance(); + continue ; + } + } } else { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a8210ea0/core/sql/executor/ExHbaseAccess.h ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExHbaseAccess.h b/core/sql/executor/ExHbaseAccess.h index 2e5da49..1c0208e 100644 --- a/core/sql/executor/ExHbaseAccess.h +++ b/core/sql/executor/ExHbaseAccess.h @@ -329,6 +329,7 @@ protected: short createDirectRowBuffer(UInt16 tuppIndex, char * tuppRow, Queue * listOfColNames, + Queue * listOfOmittedColNames, NABoolean isUpdate = FALSE, std::vector<UInt32> * posVec = NULL, double sampleRate = 0.0); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a8210ea0/core/sql/executor/ExHbaseIUD.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExHbaseIUD.cpp b/core/sql/executor/ExHbaseIUD.cpp index 8405089..522c741 100644 --- a/core/sql/executor/ExHbaseIUD.cpp +++ b/core/sql/executor/ExHbaseIUD.cpp @@ -494,6 +494,7 @@ ExWorkProcRetcode ExHbaseAccessInsertSQTcb::work() retcode = createDirectRowBuffer( hbaseAccessTdb().convertTuppIndex_, convertRow_, hbaseAccessTdb().listOfUpdatedColNames(), + hbaseAccessTdb().listOfOmittedColNames(), (hbaseAccessTdb().hbaseSqlIUD() ? FALSE : TRUE)); if (retcode == -1) @@ -880,6 +881,7 @@ ExWorkProcRetcode ExHbaseAccessUpsertVsbbSQTcb::work() hbaseAccessTdb().convertTuppIndex_, convertRow_, hbaseAccessTdb().listOfUpdatedColNames(), + hbaseAccessTdb().listOfOmittedColNames(), TRUE); if (retcode == -1) { @@ -1492,6 +1494,7 @@ ExWorkProcRetcode ExHbaseAccessBulkLoadPrepSQTcb::work() hbaseAccessTdb().convertTuppIndex_, convertRow_, hbaseAccessTdb().listOfUpdatedColNames(), + hbaseAccessTdb().listOfOmittedColNames(), FALSE, //TRUE, &posVec_, samplingRate); @@ -2001,6 +2004,7 @@ ExWorkProcRetcode ExHbaseUMDtrafUniqueTaskTcb::work(short &rc) retcode = tcb_->createDirectRowBuffer( tcb_->hbaseAccessTdb().updateTuppIndex_, tcb_->updateRow_, tcb_->hbaseAccessTdb().listOfUpdatedColNames(), + tcb_->hbaseAccessTdb().listOfOmittedColNames(), TRUE); if (retcode == -1) { @@ -2059,7 +2063,8 @@ ExWorkProcRetcode ExHbaseUMDtrafUniqueTaskTcb::work(short &rc) retcode = tcb_->createDirectRowBuffer( tcb_->hbaseAccessTdb().mergeInsertTuppIndex_, tcb_->mergeInsertRow_, - tcb_->hbaseAccessTdb().listOfMergedColNames()); + tcb_->hbaseAccessTdb().listOfMergedColNames(), + tcb_->hbaseAccessTdb().listOfOmittedColNames()); if (retcode == -1) { step_ = HANDLE_ERROR; @@ -2934,6 +2939,7 @@ ExWorkProcRetcode ExHbaseUMDtrafSubsetTaskTcb::work(short &rc) tcb_->hbaseAccessTdb().updateTuppIndex_, tcb_->updateRow_, tcb_->hbaseAccessTdb().listOfUpdatedColNames(), + tcb_->hbaseAccessTdb().listOfOmittedColNames(), TRUE); if (retcode == -1) { @@ -4166,7 +4172,8 @@ ExWorkProcRetcode ExHbaseAccessSQRowsetTcb::work() retcode = createDirectRowBuffer( hbaseAccessTdb().updateTuppIndex_, updateRow_, - hbaseAccessTdb().listOfUpdatedColNames(), + hbaseAccessTdb().listOfUpdatedColNames(), + hbaseAccessTdb().listOfOmittedColNames(), TRUE); if (retcode == -1) { step_ = HANDLE_ERROR; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a8210ea0/core/sql/generator/GenPreCode.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenPreCode.cpp b/core/sql/generator/GenPreCode.cpp index b7c9ef2..e6a3e01 100644 --- a/core/sql/generator/GenPreCode.cpp +++ b/core/sql/generator/GenPreCode.cpp @@ -4504,19 +4504,6 @@ RelExpr * UpdateCursor::preCodeGen(Generator * generator, { ItemExpr * item_expr = val_id.getItemExpr(); - if (isMerge()) - { - // column being updated must be from the same table that is being - // updated. - if (((BaseColumn *)(item_expr->child(0)->castToItemExpr()))->getTableDesc() != - getTableDesc()) - { - *CmpCommon::diags() << DgSqlCode(-3241) - << DgString0("Invalid column being updated."); - GenExit(); - } - } - for (short i = 0; i < getTableDesc()->getNATable()->getKeyCount(); i++) { const char * key_colname = key_column_array[i]->getColName(); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a8210ea0/core/sql/generator/GenRelUpdate.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelUpdate.cpp b/core/sql/generator/GenRelUpdate.cpp index da7c8ea..e544c36 100644 --- a/core/sql/generator/GenRelUpdate.cpp +++ b/core/sql/generator/GenRelUpdate.cpp @@ -469,7 +469,8 @@ static short genHbaseUpdOrInsertExpr( listOfUpdatedColNames = NULL; if (updRecExprArray.entries() > 0) listOfUpdatedColNames = new(space) Queue(space); - + + for (CollIndex ii = 0; ii < updRecExprArray.entries(); ii++) { const ItemExpr *assignExpr = updRecExprArray[ii].getItemExpr(); @@ -559,7 +560,6 @@ static short genHbaseUpdOrInsertExpr( Attributes * updValAttr = (generator->getMapInfo(updValId, 0))->getAttr(); assignAttr->copyLocationAttrs(updValAttr); } - for (CollIndex ii = 0; ii < updRecExprArray.entries(); ii++) { const ItemExpr *assignExpr = updRecExprArray[ii].getItemExpr(); @@ -832,8 +832,8 @@ short HbaseDelete::codeGen(Generator * generator) ValueIdList srcVIDlist; ValueIdList dupVIDlist; HbaseAccess::sortValues(retColRefSet_, - columnList, - srcVIDlist, dupVIDlist, + columnList, + srcVIDlist, dupVIDlist, (getTableDesc()->getNATable()->getExtendedQualName().getSpecialType() == ExtendedQualName::INDEX_TABLE)); const CollIndex numColumns = columnList.entries(); @@ -906,7 +906,7 @@ short HbaseDelete::codeGen(Generator * generator) givenType, // [IN] Actual type of HDFS column asciiValue, // [OUT] Returned expression for ascii rep. castValue, // [OUT] Returned expression for binary rep. - isAlignedFormat + isAlignedFormat ); GenAssert(res == 1 && asciiValue != NULL && castValue != NULL, @@ -939,9 +939,9 @@ short HbaseDelete::codeGen(Generator * generator) asciiTuppIndex, // [IN] index into atp &asciiTupleDesc, // [optional OUT] tuple desc ExpTupleDesc::LONG_FORMAT, // [optional IN] desc format - 0, - NULL, - (NAColumnArray*)colArray); + 0, + NULL, + (NAColumnArray*)colArray); work_cri_desc->setTupleDescriptor(asciiTuppIndex, asciiTupleDesc); @@ -1033,10 +1033,10 @@ short HbaseDelete::codeGen(Generator * generator) if (addDefaultValues) //hasAddedColumns) { expGen->addDefaultValues(columnList, - getIndexDesc()->getAllColumns(), - tuple_desc, - TRUE); - + getIndexDesc()->getAllColumns(), + tuple_desc, + TRUE); + if (asciiRowFormat == ExpTupleDesc::SQLMX_ALIGNED_FORMAT) { expGen->addDefaultValues(columnList, @@ -1814,6 +1814,8 @@ short HbaseUpdate::codeGen(Generator * generator) ULng32 returnedFetchedRowLen = 0; ULng32 returnedUpdatedRowLen = 0; ULng32 returnedMergeInsertedRowLen = 0; + Queue *listOfOmittedColNames = NULL; + if (returnRow) { const ValueIdList &fetchedOutputs = @@ -1968,18 +1970,35 @@ short HbaseUpdate::codeGen(Generator * generator) expGen->assignAtpAndAtpIndex(getScanIndexDesc()->getIndexColumns(), 0, returnedFetchedTuppIndex); */ - + NAColumn *col; if (isMerge()) { ValueIdList mergeInsertOutputs; for (CollIndex ii = 0; ii < mergeInsertRecExprArray().entries(); ii++) - { + { const ItemExpr *assignExpr = mergeInsertRecExprArray()[ii].getItemExpr(); ValueId tgtValueId = assignExpr->child(0)->castToItemExpr()->getValueId(); - + col = tgtValueId.getNAColumn( TRUE ); + + if ((NOT isAlignedFormat) && + (((Assign*)assignExpr)->canBeSkipped()) && + (NOT col->isSystemColumn()) && + (NOT col->isClusteringKey()) && + (NOT col->isIdentityColumn())) + { + if (listOfOmittedColNames == NULL) + listOfOmittedColNames = new(space) Queue(space); + NAString cnInList; + HbaseAccess::createHbaseColId(col, cnInList); + + char * colNameInList = + space->AllocateAndCopyToAlignedSpace(cnInList, 0); + + listOfOmittedColNames->insert(colNameInList); + } mergeInsertOutputs.insert(tgtValueId); - } + } MapTable * returnedMergeInsertedMapTable = 0; ExpTupleDesc * returnedMergeInsertedTupleDesc = NULL; @@ -2136,6 +2155,7 @@ short HbaseUpdate::codeGen(Generator * generator) ); generator->initTdbFields(hbasescan_tdb); + hbasescan_tdb->setListOfOmittedColNames(listOfOmittedColNames); if (getTableDesc()->getNATable()->isHbaseRowTable()) hbasescan_tdb->setRowwiseFormat(TRUE); @@ -2256,7 +2276,6 @@ short HbaseInsert::codeGen(Generator *generator) ULng32 loggingRowLen = 0; - NABoolean addDefaultValues = TRUE; NABoolean hasAddedColumns = FALSE; if (getTableDesc()->getNATable()->hasAddedColumn()) hasAddedColumns = TRUE; @@ -2267,34 +2286,37 @@ short HbaseInsert::codeGen(Generator *generator) NAColumn *col; ValueIdList returnRowVIDList; - NABoolean upsertColsWereSkipped = FALSE; + Queue *listOfOmittedColNames = NULL; + NABoolean isAlignedFormat = getTableDesc()->getNATable()->isAlignedFormat(getIndexDesc()); for (CollIndex ii = 0; ii < newRecExprArray().entries(); ii++) - { + { const ItemExpr *assignExpr = newRecExprArray()[ii].getItemExpr(); ValueId tgtValueId = assignExpr->child(0)->castToItemExpr()->getValueId(); ValueId srcValueId = assignExpr->child(1)->castToItemExpr()->getValueId(); col = tgtValueId.getNAColumn( TRUE ); + if ((NOT isAlignedFormat) && + (((Assign*)assignExpr)->canBeSkipped()) && + (NOT col->isSystemColumn()) && + (NOT col->isClusteringKey()) && + (NOT col->isIdentityColumn())) + { + if (listOfOmittedColNames == NULL) + listOfOmittedColNames = new(space) Queue(space); + NAString cnInList; + HbaseAccess::createHbaseColId(col, cnInList); - // if upsert stmt and this assign was not specified by user, skip it. - // If used, it will overwrite existing values if this row exists in the table. - if ((isUpsert()) && - (NOT ((Assign*)assignExpr)->isUserSpecified()) && - (NOT col->isSystemColumn()) && - (NOT col->isIdentityColumn())) - { - upsertColsWereSkipped = TRUE; - continue; - } + char * colNameInList = + space->AllocateAndCopyToAlignedSpace(cnInList, 0); + listOfOmittedColNames->insert(colNameInList); + } + colArray.insert( col ); if (returnRow) - returnRowVIDList.insert(tgtValueId); + returnRowVIDList.insert(tgtValueId); - if ( col != NULL ) - colArray.insert( col ); - ItemExpr * child1Expr = assignExpr->child(1); const NAType &givenType = tgtValueId.getType(); @@ -2313,9 +2335,8 @@ short HbaseInsert::codeGen(Generator *generator) { GenAssert(0,"bindNode failed"); } - insertVIDList.insert(ie->getValueId()); - } + } const NATable *naTable = getTableDesc()->getNATable(); @@ -2345,13 +2366,6 @@ short HbaseInsert::codeGen(Generator *generator) NULL, NULL, &colArray); - - if (addDefaultValues) - { - expGen->addDefaultValues(insertVIDList, - (upsertColsWereSkipped ? colArray : getIndexDesc()->getAllColumns()), - tupleDesc); - } ex_expr * loggingDataExpr = NULL; ExpTupleDesc * loggingDataTupleDesc = NULL; @@ -2417,24 +2431,13 @@ short HbaseInsert::codeGen(Generator *generator) NAList<Attributes*> savedInputAttrsList; const ValueIdList &indexVIDlist = getIndexDesc()->getIndexColumns(); - CollIndex jj = 0; for (CollIndex ii = 0; ii < newRecExprArray().entries(); ii++) { const ItemExpr *assignExpr = newRecExprArray()[ii].getItemExpr(); const ValueId &tgtValueId = assignExpr->child(0)->castToItemExpr()->getValueId(); const ValueId &indexValueId = indexVIDlist[ii]; col = tgtValueId.getNAColumn( TRUE ); - - if ((isUpsert()) && - (NOT ((Assign*)assignExpr)->isUserSpecified()) && - (NOT col->isSystemColumn()) && - (NOT col->isIdentityColumn())) - { - continue; - } - - ValueId &srcValId = insertVIDList[jj]; - jj++; + ValueId &srcValId = insertVIDList[ii]; Attributes * colAttr = (generator->addMapInfo(tgtValueId, 0))->getAttr(); Attributes * indexAttr = (generator->addMapInfo(indexValueId, 0))->getAttr(); @@ -2815,6 +2818,7 @@ short HbaseInsert::codeGen(Generator *generator) ); generator->initTdbFields(hbasescan_tdb); + hbasescan_tdb->setListOfOmittedColNames(listOfOmittedColNames); if ((CmpCommon::getDefault(HBASE_ASYNC_OPERATIONS) == DF_ON) && getInliningInfo().isIMGU()) http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a8210ea0/core/sql/optimizer/BindRelExpr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp index 6852237..c0b3788 100644 --- a/core/sql/optimizer/BindRelExpr.cpp +++ b/core/sql/optimizer/BindRelExpr.cpp @@ -9798,6 +9798,10 @@ RelExpr *Insert::bindNode(BindWA *bindWA) if (identityColumnGeneratedAlways) defaultColCount = totalColCount; + NABoolean isAlignedRowFormat = getTableDesc()->getNATable()->isSQLMXAlignedTable(); + NABoolean omittedDefaultCols = FALSE; + NABoolean omittedCurrentDefaultClassCols = FALSE; + if (defaultColCount) { NAWchar zero_w_Str[2]; zero_w_Str[0] = L'0'; zero_w_Str[1] = L'\0'; // wide version CollIndex sysColIx = 0, usrColIx = 0; @@ -9877,11 +9881,15 @@ RelExpr *Insert::bindNode(BindWA *bindWA) // if (nacol->getDefaultClass() == COM_CURRENT_DEFAULT) { castType = nacol->getType()->newCopy(bindWA->wHeap()); + omittedCurrentDefaultClassCols = TRUE; + omittedDefaultCols = TRUE; } else if ((nacol->getDefaultClass() == COM_IDENTITY_GENERATED_ALWAYS) || (nacol->getDefaultClass() == COM_IDENTITY_GENERATED_BY_DEFAULT)) { setSystemGeneratesIdentityValue(TRUE); } + else + omittedDefaultCols = TRUE; // Bind the default value, make an Assign, etc, as above Parser parser(bindWA->currentCmpContext()); @@ -9942,10 +9950,11 @@ RelExpr *Insert::bindNode(BindWA *bindWA) return boundExpr; } - assign = new (bindWA->wHeap()) Assign(target.getItemExpr(), defaultValueExpr, - FALSE /*not user-specified*/); + FALSE /*Not user Specified */); + if (nacol->getDefaultClass() != COM_CURRENT_DEFAULT) + assign->setToBeSkipped(TRUE); assign->bindNode(bindWA); } @@ -10102,8 +10111,15 @@ RelExpr *Insert::bindNode(BindWA *bindWA) // 4490 - BULK LOAD into a salted table is not supported if ESP parallelism is turned off *CmpCommon::diags() << DgSqlCode(-4490); } - } + if (isUpsert() && (!getIsTrafLoadPrep()) && + (((!isAlignedRowFormat) && omittedCurrentDefaultClassCols) || + (isAlignedRowFormat && omittedDefaultCols)) + && (CmpCommon::getDefault(TRAF_UPSERT_WITH_INSERT_DEFAULT_SEMANTICS) == DF_OFF)) { + boundExpr = xformUpsertToMerge(bindWA); + return boundExpr; + } + else if (isUpsertThatNeedsMerge()) { boundExpr = xformUpsertToMerge(bindWA); return boundExpr; @@ -10167,9 +10183,9 @@ NABoolean Insert::isUpsertThatNeedsMerge() const return TRUE; } + RelExpr* Insert::xformUpsertToMerge(BindWA *bindWA) { - NATable *naTable = bindWA->getNATable(getTableName()); if (bindWA->errStatus()) return NULL; @@ -10183,6 +10199,8 @@ RelExpr* Insert::xformUpsertToMerge(BindWA *bindWA) const ValueIdList &tableCols = updateToSelectMap().getTopValues(); const ValueIdList &sourceVals = updateToSelectMap().getBottomValues(); + + NABoolean isAlignedRowFormat = getTableDesc()->getNATable()->isSQLMXAlignedTable(); Scan * inputScan = new (bindWA->wHeap()) @@ -10225,17 +10243,6 @@ RelExpr* Insert::xformUpsertToMerge(BindWA *bindWA) keyPred); } } - else - { - setAssignPrev = setAssign; - setAssign = new (bindWA->wHeap()) - Assign(targetColRef, sourceVals[i].getItemExpr()); - setCount++; - if (setCount > 1) - { - setAssign = new(bindWA->wHeap()) ItemList(setAssign,setAssignPrev); - } - } if (sourceVals[i].getItemExpr()->getOperatorType() != ITM_CONSTANT) myOuterRefs += sourceVals[i]; @@ -10250,8 +10257,40 @@ RelExpr* Insert::xformUpsertToMerge(BindWA *bindWA) insertVal = new(bindWA->wHeap()) ItemList(insertVal,insertValPrev); insertCol = new(bindWA->wHeap()) ItemList(insertCol,insertColPrev); } - } + } inputScan->addSelPredTree(keyPred); + for (CollIndex i = 0 ; i < newRecExprArray().entries(); i++) + { + const Assign *assignExpr = (Assign *)newRecExprArray()[i].getItemExpr(); + ValueId tgtValueId = assignExpr->child(0)->castToItemExpr()->getValueId(); + NAColumn *col = tgtValueId.getNAColumn( TRUE ); + NABoolean copySetAssign = FALSE; + if (! col->isClusteringKey()) + { + // In case of aligned format we need to bind in the new = old values + // in GenericUpdate::bindNode. So skip the columns that are not user + // specified + if (isAlignedRowFormat) + { + if (assignExpr->isUserSpecified()) + copySetAssign = TRUE; + // copy the default value if the below CQD is set to ON + else if (CmpCommon::getDefault(TRAF_UPSERT_WITH_INSERT_DEFAULT_SEMANTICS) == DF_ON) + copySetAssign = TRUE; + } + else + if (assignExpr->isUserSpecified()) + copySetAssign = TRUE; + if (copySetAssign) + { + setAssignPrev = setAssign; + setAssign = (ItemExpr *)assignExpr; + setCount++; + if (setCount > 1) + setAssign = new(bindWA->wHeap()) ItemList(setAssign,setAssignPrev); + } + } + } RelExpr * re = NULL; re = new (bindWA->wHeap()) @@ -10288,7 +10327,15 @@ RelExpr* Insert::xformUpsertToMerge(BindWA *bindWA) re = re->bindNode(bindWA); if (bindWA->errStatus()) return NULL; - + // Copy the userSecified and canBeSkipped attribute to mergeUpdateInsertExprArray + ValueIdList mergeInsertExprArray = ((MergeUpdate *)mu)->mergeInsertRecExprArray(); + for (CollIndex i = 0 ; i < newRecExprArray().entries(); i++) + { + const Assign *assignExpr = (Assign *)newRecExprArray()[i].getItemExpr(); + ((Assign *)mergeInsertExprArray[i].getItemExpr())->setToBeSkipped(assignExpr->canBeSkipped()); + ((Assign *)mergeInsertExprArray[i].getItemExpr())->setUserSpecified(assignExpr->isUserSpecified()); + } + return re; } @@ -12703,6 +12750,7 @@ RelExpr *GenericUpdate::bindNode(BindWA *bindWA) } } // REL_UNARY_UPDATE or REL_UNARY_DELETE + // QSTUFF // we need to check whether this code is executed as part of a create view // ddl operation using bindWA->inDDL() and prevent indices, contraints and @@ -12838,7 +12886,6 @@ RelExpr *LeafInsert::bindNode(BindWA *bindWA) Assign *assign; assign = new (bindWA->wHeap()) Assign(tgtcols[i].getItemExpr(), baseColRefs()[i], FALSE); - assign->bindNode(bindWA); if (bindWA->errStatus()) return NULL; newRecExprArray().insertAt(i, assign->getValueId()); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a8210ea0/core/sql/optimizer/ItemOther.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/ItemOther.h b/core/sql/optimizer/ItemOther.h index 7f6bfc1..a1e045e 100644 --- a/core/sql/optimizer/ItemOther.h +++ b/core/sql/optimizer/ItemOther.h @@ -80,6 +80,7 @@ public: NABoolean userSpecified = TRUE) : ItemExpr(ITM_ASSIGN, target, source), userSpecified_(userSpecified), + canBeSkipped_(FALSE), onRollback_(FALSE) {} @@ -90,6 +91,10 @@ public: virtual Int32 getArity() const; NABoolean isUserSpecified() const { return userSpecified_; } + void setUserSpecified(NABoolean userSpecified) { userSpecified_ = userSpecified; } + + NABoolean canBeSkipped() const { return canBeSkipped_; } + void setToBeSkipped(NABoolean canBeSkipped) { canBeSkipped_ = canBeSkipped; } ValueId getTarget() const { return child(0)->getValueId(); } ValueId getSource() const { return child(1)->getValueId(); } @@ -164,6 +169,8 @@ private: // Do the actual type propagation, to be called by the public synthesizeType methods. const NAType *doSynthesizeType(ValueId & targetId, ValueId & sourceId); + // Assign with omitted default value columns other than COM_CURRENT_DEFAULT class types + NABoolean canBeSkipped_; }; // class Assign http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a8210ea0/core/sql/optimizer/RelExpr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelExpr.cpp b/core/sql/optimizer/RelExpr.cpp index 7a4bbc4..2b21ae4 100644 --- a/core/sql/optimizer/RelExpr.cpp +++ b/core/sql/optimizer/RelExpr.cpp @@ -12797,7 +12797,7 @@ MergeUpdate::MergeUpdate(const CorrName &name, ItemExpr *where) : Update(name,tabId,otype,child,setExpr,NULL,oHeap), insertCols_(insertCols), insertValues_(insertValues), - where_(where),xformedUpsert_(FALSE) + where_(where) { setCacheableNode(CmpMain::BIND); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a8210ea0/core/sql/optimizer/RelUpdate.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelUpdate.h b/core/sql/optimizer/RelUpdate.h index ccb00dd..3eea633 100644 --- a/core/sql/optimizer/RelUpdate.h +++ b/core/sql/optimizer/RelUpdate.h @@ -143,7 +143,8 @@ GenericUpdate(const CorrName &name, noIMneeded_(FALSE), useMVCC_(FALSE), useSSCC_(FALSE), - preconditionTree_(NULL) + preconditionTree_(NULL), + xformedUpsert_(FALSE) {} // copy ctor @@ -541,6 +542,8 @@ GenericUpdate(const CorrName &name, { precondition_ = pc; exprsInDerivedClasses_ += precondition_; } virtual ItemExpr * insertValues() { return NULL;} + NABoolean xformedUpsert() {return xformedUpsert_;} + void setXformedUpsert() {xformedUpsert_ = TRUE;} protected: @@ -928,6 +931,8 @@ private: // LeafDelete (IM). Only insert/delete if TRUE ItemExpr *preconditionTree_; ValueIdSet precondition_; + + NABoolean xformedUpsert_; }; @@ -1441,13 +1446,10 @@ public: ItemExpr * insertCols() {return insertCols_;} virtual ItemExpr * insertValues() { return insertValues_;} - NABoolean xformedUpsert() {return xformedUpsert_;} - void setXformedUpsert() {xformedUpsert_ = TRUE;} private: ItemExpr *insertCols_; ItemExpr *insertValues_; ItemExpr *where_; - NABoolean xformedUpsert_; }; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a8210ea0/core/sql/sqlcomp/DefaultConstants.h ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/DefaultConstants.h b/core/sql/sqlcomp/DefaultConstants.h index d4acd8d..14b7ee5 100644 --- a/core/sql/sqlcomp/DefaultConstants.h +++ b/core/sql/sqlcomp/DefaultConstants.h @@ -3796,6 +3796,9 @@ enum DefaultConstants // real charset in the HIVE table HIVE_FILE_CHARSET, + // if ON, upsert into the table will use the default value for the omitted columns + // with default value + TRAF_UPSERT_WITH_INSERT_DEFAULT_SEMANTICS, // This enum constant must be the LAST one in the list; it's a count, // not an Attribute (it's not IN DefaultDefaults; it's the SIZE of it)! __NUM_DEFAULT_ATTRIBUTES http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a8210ea0/core/sql/sqlcomp/nadefaults.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp index b556100..bd1761a 100644 --- a/core/sql/sqlcomp/nadefaults.cpp +++ b/core/sql/sqlcomp/nadefaults.cpp @@ -3400,6 +3400,7 @@ XDDkwd__(SUBQUERY_UNNESTING, "ON"), DDkwd__(TRAF_UPSERT_ADJUST_PARAMS, "OFF"), DDkwd__(TRAF_UPSERT_AUTO_FLUSH, "OFF"), DDint__(TRAF_UPSERT_WB_SIZE, "2097152"), + DDkwd__(TRAF_UPSERT_WITH_INSERT_DEFAULT_SEMANTICS, "OFF"), DDkwd__(TRAF_UPSERT_WRITE_TO_WAL, "OFF"), DDkwd__(TRAF_USE_RWRS_FOR_MD_INSERT, "ON"),
