Repository: incubator-trafodion Updated Branches: refs/heads/master 67288b3cd -> 940ad6f4c
[TRAFODION-2438] Unnecessary and sometimes wrong delete during upsert/merge with IM This change is done by Hans. Design by Selva. Please see JIRA for an explanation. Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/1450e836 Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/1450e836 Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/1450e836 Branch: refs/heads/master Commit: 1450e83627b5bdde6f947c963152ce00f9d5e907 Parents: c572f63 Author: Suresh Subbiah <[email protected]> Authored: Fri Jan 20 18:15:06 2017 +0000 Committer: Suresh Subbiah <[email protected]> Committed: Fri Jan 20 18:15:06 2017 +0000 ---------------------------------------------------------------------- core/sql/comexe/ComTdbHbaseAccess.cpp | 5 +- core/sql/comexe/ComTdbHbaseAccess.h | 2 + core/sql/executor/ExHbaseAccess.cpp | 3 + core/sql/executor/ExHbaseIUD.cpp | 24 ++++-- core/sql/generator/GenRelScan.cpp | 2 + core/sql/generator/GenRelUpdate.cpp | 49 ++++++++++- core/sql/optimizer/BindRelExpr.cpp | 119 ++++++++++++++++++-------- core/sql/optimizer/ImplRule.cpp | 4 +- core/sql/optimizer/Inlining.cpp | 54 +++++++++--- core/sql/optimizer/NormRelExpr.cpp | 11 +++ core/sql/optimizer/RETDesc.cpp | 44 +++++++--- core/sql/optimizer/RelExpr.cpp | 6 +- core/sql/optimizer/RelJoin.h | 2 + core/sql/optimizer/RelUpdate.h | 74 ++++++++++++++-- core/sql/regress/executor/EXPECTED015.SB | 39 +++++++++ core/sql/regress/executor/TEST015 | 7 ++ 16 files changed, 365 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1450e836/core/sql/comexe/ComTdbHbaseAccess.cpp ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbHbaseAccess.cpp b/core/sql/comexe/ComTdbHbaseAccess.cpp index 676f2b2..1e81d86 100644 --- a/core/sql/comexe/ComTdbHbaseAccess.cpp +++ b/core/sql/comexe/ComTdbHbaseAccess.cpp @@ -66,6 +66,7 @@ ComTdbHbaseAccess::ComTdbHbaseAccess( const UInt16 updateTuppIndex, const UInt16 mergeInsertTuppIndex, const UInt16 mergeInsertRowIdTuppIndex, + const UInt16 mergeIUDIndicatorTuppIndex, const UInt16 returnedFetchedTuppIndex, const UInt16 returnedUpdatedTuppIndex, @@ -156,6 +157,7 @@ ComTdbHbaseAccess::ComTdbHbaseAccess( updateTuppIndex_(updateTuppIndex), mergeInsertTuppIndex_(mergeInsertTuppIndex), mergeInsertRowIdTuppIndex_(mergeInsertRowIdTuppIndex), + mergeIUDIndicatorTuppIndex_(mergeIUDIndicatorTuppIndex), returnedFetchedTuppIndex_(returnedFetchedTuppIndex), returnedUpdatedTuppIndex_(returnedUpdatedTuppIndex), @@ -268,6 +270,7 @@ ComTdbHbaseAccess::ComTdbHbaseAccess( updateTuppIndex_(0), mergeInsertTuppIndex_(0), mergeInsertRowIdTuppIndex_(0), + mergeIUDIndicatorTuppIndex_(0), returnedFetchedTuppIndex_(0), returnedUpdatedTuppIndex_(0), @@ -1182,7 +1185,7 @@ ComTdbHbaseCoProcAccess::ComTdbHbaseCoProcAccess( projExpr, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0, projRowLen, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, projTuppIndex, 0, 0, 0, 0, 0, 0, returnedTuppIndex, 0, 0, 0, 0, 0, 0, + 0, projTuppIndex, 0, 0, 0, 0, 0, 0, 0, returnedTuppIndex, 0, 0, 0, 0, 0, 0, NULL, NULL, listOfColNames, NULL, NULL, NULL, NULL, workCriDesc, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1450e836/core/sql/comexe/ComTdbHbaseAccess.h ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbHbaseAccess.h b/core/sql/comexe/ComTdbHbaseAccess.h index c0c3f1f..2e13a9f 100644 --- a/core/sql/comexe/ComTdbHbaseAccess.h +++ b/core/sql/comexe/ComTdbHbaseAccess.h @@ -452,6 +452,7 @@ public: const UInt16 updateTuppIndex, const UInt16 mergeInsertTuppIndex, const UInt16 mergeInsertRowIdTuppIndex, + const UInt16 mergeIUDIndicatorTuppIndex, const UInt16 returnedFetchedTuppIndex, const UInt16 returnedUpdatedTuppIndex, @@ -950,6 +951,7 @@ public: UInt16 updateTuppIndex_; UInt16 mergeInsertTuppIndex_; UInt16 mergeInsertRowIdTuppIndex_; + UInt16 mergeIUDIndicatorTuppIndex_; UInt16 returnedFetchedTuppIndex_; UInt16 returnedUpdatedTuppIndex_; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1450e836/core/sql/executor/ExHbaseAccess.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExHbaseAccess.cpp b/core/sql/executor/ExHbaseAccess.cpp index cb19887..a414f2e 100644 --- a/core/sql/executor/ExHbaseAccess.cpp +++ b/core/sql/executor/ExHbaseAccess.cpp @@ -269,6 +269,9 @@ ExHbaseAccessTcb::ExHbaseAccessTcb( pool_->get_free_tuple(workAtp_->getTupp(hbaseAccessTdb.mergeInsertTuppIndex_), 0); if (hbaseAccessTdb.mergeInsertRowIdTuppIndex_ > 0) pool_->get_free_tuple(workAtp_->getTupp(hbaseAccessTdb.mergeInsertRowIdTuppIndex_), 0); + if (hbaseAccessTdb.mergeIUDIndicatorTuppIndex_ > 0) + pool_->get_free_tuple(workAtp_->getTupp(hbaseAccessTdb.mergeIUDIndicatorTuppIndex_), 0); + if (hbaseAccessTdb.rowIdTuppIndex_ > 0) pool_->get_free_tuple(workAtp_->getTupp(hbaseAccessTdb.rowIdTuppIndex_), 0); if (hbaseAccessTdb.rowIdAsciiTuppIndex_ > 0) http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1450e836/core/sql/executor/ExHbaseIUD.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExHbaseIUD.cpp b/core/sql/executor/ExHbaseIUD.cpp index 849e28f..d87db65 100644 --- a/core/sql/executor/ExHbaseIUD.cpp +++ b/core/sql/executor/ExHbaseIUD.cpp @@ -2389,6 +2389,7 @@ ExWorkProcRetcode ExHbaseUMDtrafUniqueTaskTcb::work(short &rc) char * fetchedDataPtr = NULL; char * updatedDataPtr = NULL; + char * mergeIUDIndicatorDataPtr = NULL; if (tcb_->returnFetchExpr()) { exprRetCode = @@ -2401,11 +2402,18 @@ ExWorkProcRetcode ExHbaseUMDtrafUniqueTaskTcb::work(short &rc) fetchedDataPtr = up_entry->getAtp()->getTupp(tcb_->hbaseAccessTdb().returnedFetchedTuppIndex_).getDataPointer(); } - + if (tcb_->hbaseAccessTdb().mergeIUDIndicatorTuppIndex_ > 0) + mergeIUDIndicatorDataPtr = + tcb_->workAtp_-> + getTupp(tcb_->hbaseAccessTdb().mergeIUDIndicatorTuppIndex_). + getDataPointer(); + if (rowUpdated_) { if (tcb_->returnUpdateExpr()) { + if (mergeIUDIndicatorDataPtr) + *mergeIUDIndicatorDataPtr = 'U'; exprRetCode = tcb_->returnUpdateExpr()->eval(up_entry->getAtp(), tcb_->workAtp_); if (exprRetCode == ex_expr::EXPR_ERROR) @@ -2419,6 +2427,8 @@ ExWorkProcRetcode ExHbaseUMDtrafUniqueTaskTcb::work(short &rc) } else { + if (mergeIUDIndicatorDataPtr) + *mergeIUDIndicatorDataPtr = 'I'; if (tcb_->returnMergeInsertExpr()) { exprRetCode = @@ -4022,12 +4032,6 @@ ExWorkProcRetcode ExHbaseAccessSQRowsetTcb::work() break; case SETUP_UMD: { - rowIds_.clear(); - retcode = setupUniqueKeyAndCols(FALSE); - if (retcode == -1) { - step_ = HANDLE_ERROR; - break; - } rc = evalInsDelPreCondExpr(); if (rc == -1) { step_ = HANDLE_ERROR; @@ -4037,6 +4041,12 @@ ExWorkProcRetcode ExHbaseAccessSQRowsetTcb::work() step_ = NEXT_ROW; break; } + rowIds_.clear(); + retcode = setupUniqueKeyAndCols(FALSE); + if (retcode == -1) { + step_ = HANDLE_ERROR; + break; + } copyRowIDToDirectBuffer(rowIds_[0]); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1450e836/core/sql/generator/GenRelScan.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelScan.cpp b/core/sql/generator/GenRelScan.cpp index c027916..5dbc237 100644 --- a/core/sql/generator/GenRelScan.cpp +++ b/core/sql/generator/GenRelScan.cpp @@ -3176,6 +3176,7 @@ short HbaseAccess::codeGen(Generator * generator) 0, // updateTuppIndex 0, // mergeInsertTuppIndex 0, // mergeInsertRowIdTuppIndex + 0, // mergeIUDIndicatorTuppIndex 0, // returnedFetchedTuppIndex 0, // returnedUpdatedTuppIndex @@ -3436,6 +3437,7 @@ short HbaseAccessCoProcAggr::codeGen(Generator * generator) 0, // updateTuppIndex 0, // mergeInsertTuppIndex 0, // mergeInsertRowIdTuppIndex + 0, // mergeIUDIndicatorTuppIndex 0, // returnedFetchedTuppIndex 0, // returnedUpdatedTuppIndex http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1450e836/core/sql/generator/GenRelUpdate.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelUpdate.cpp b/core/sql/generator/GenRelUpdate.cpp index f3dbf58..a6ac674 100644 --- a/core/sql/generator/GenRelUpdate.cpp +++ b/core/sql/generator/GenRelUpdate.cpp @@ -1153,6 +1153,7 @@ short HbaseDelete::codeGen(Generator * generator) 0, // updateTuppIndex 0, // mergeInsertTuppIndex 0, // mergeInsertRowIdTuppIndex + 0, // mergeIUDIndicatorTuppIndex 0, // returnedFetchedTuppIndex 0, // returnedUpdatedTuppIndex @@ -1319,12 +1320,41 @@ short HbaseUpdate::codeGen(Generator * generator) const Int32 mergeInsertTuppIndex = 7; const Int32 mergeInsertRowIdTuppIndex = 8; const Int32 keyColValTuppIndex = 9; + Int32 mergeIUDIndicatorTuppIndex = 0; + // Do not use 10 as the next available tupp index. Please use 11 next + // The 10th tuple index is used by merge statement below. + + Attributes * iudIndicatorAttr = NULL; ULng32 asciiRowLen = 0; ExpTupleDesc * asciiTupleDesc = 0; ex_cri_desc * work_cri_desc = NULL; - work_cri_desc = new(space) ex_cri_desc(10, space); + work_cri_desc = new(space) ex_cri_desc(11, space); + + if (getProducedMergeIUDIndicator() != NULL_VALUE_ID) + { + mergeIUDIndicatorTuppIndex = 10; + iudIndicatorAttr = + (generator->addMapInfo(getProducedMergeIUDIndicator(), 0))->getAttr(); + iudIndicatorAttr->setAtpIndex(mergeIUDIndicatorTuppIndex); + iudIndicatorAttr->setAtp(work_atp); + ULng32 iudIndicatorLen; + ExpTupleDesc::computeOffsets(iudIndicatorAttr, + ExpTupleDesc::SQLARK_EXPLODED_FORMAT, + iudIndicatorLen); + ExpTupleDesc *iudIndicatorTupleDesc = NULL; + iudIndicatorTupleDesc = new(generator->getSpace()) + ExpTupleDesc(1, // numAttrs + &iudIndicatorAttr, // **attrs + iudIndicatorLen, // data length + ExpTupleDesc::SQLARK_EXPLODED_FORMAT, + ExpTupleDesc::LONG_FORMAT, + generator->getSpace()); + work_cri_desc->setTupleDescriptor(mergeIUDIndicatorTuppIndex, + iudIndicatorTupleDesc); + + } NABoolean returnRow = getReturnRow(this, getIndexDesc()); @@ -1812,6 +1842,8 @@ short HbaseUpdate::codeGen(Generator * generator) updatedOutputs.insert(tgtValueId); } + if (getProducedMergeIUDIndicator() != NULL_VALUE_ID) + updatedOutputs.insert(getProducedMergeIUDIndicator()); } else { @@ -1865,12 +1897,19 @@ short HbaseUpdate::codeGen(Generator * generator) &tgtConvValueIdList); } + Attributes * tgtIUDMergeColConvAttr = NULL; const ValueIdList &indexColList = getIndexDesc()->getIndexColumns(); for (CollIndex ii = 0; ii < tgtConvValueIdList.entries(); ii++) { const ValueId &tgtColValueId = updatedOutputs[ii]; const ValueId &tgtColConvValueId = tgtConvValueIdList[ii]; - + if (updatedOutputs[ii] == getProducedMergeIUDIndicator()) + { + tgtIUDMergeColConvAttr = + (generator->getMapInfo(tgtColConvValueId, 0))->getAttr(); + continue; + } + BaseColumn * bc = (BaseColumn*)tgtColValueId.getItemExpr(); const ValueId &indexColValueId = indexColList[bc->getColNumber()]; @@ -1922,6 +1961,8 @@ short HbaseUpdate::codeGen(Generator * generator) } mergeInsertOutputs.insert(tgtValueId); } + if (getProducedMergeIUDIndicator() != NULL_VALUE_ID) + mergeInsertOutputs.insert(getProducedMergeIUDIndicator()); MapTable * returnedMergeInsertedMapTable = 0; ExpTupleDesc * returnedMergeInsertedTupleDesc = NULL; @@ -1958,6 +1999,8 @@ short HbaseUpdate::codeGen(Generator * generator) &returnedMergeInsertedMapTable, &tgtConvValueIdList); } + if (getProducedMergeIUDIndicator() != NULL_VALUE_ID) + iudIndicatorAttr->copyLocationAttrs(tgtIUDMergeColConvAttr); } } @@ -2041,6 +2084,7 @@ short HbaseUpdate::codeGen(Generator * generator) updateTuppIndex, mergeInsertTuppIndex, mergeInsertRowIdTuppIndex, + mergeIUDIndicatorTuppIndex, returnedFetchedTuppIndex, returnedUpdatedTuppIndex, @@ -2727,6 +2771,7 @@ short HbaseInsert::codeGen(Generator *generator) loggingTuppIndex, //loggingTuppIndex 0, // mergeInsertTuppIndex 0, // mergeInsertRowIdTuppIndex + 0, // mergeIUDIndicatorTuppIndex 0, // returnedFetchedTuppIndex projRowTuppIndex, // returnedUpdatedTuppIndex http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1450e836/core/sql/optimizer/BindRelExpr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp index d51624d..0231519 100644 --- a/core/sql/optimizer/BindRelExpr.cpp +++ b/core/sql/optimizer/BindRelExpr.cpp @@ -2623,7 +2623,7 @@ RelExpr *Join::bindNode(BindWA *bindWA) RelExpr *leftJoin = this; leftJoin->setOperatorType(REL_LEFT_JOIN); - RelExpr *antiJoin = leftJoin->copyTree(bindWA->wHeap()); + Join *antiJoin = static_cast<Join *>(leftJoin->copyTree(bindWA->wHeap())); antiJoin->setOperatorType(REL_RIGHT_JOIN); NAString leftName("ALJ", bindWA->wHeap()); @@ -2640,31 +2640,7 @@ RelExpr *Join::bindNode(BindWA *bindWA) unionAll->bindNode(bindWA); if (bindWA->errStatus()) return this; - // Make sure there is at least one null instantiated - // value that is suitable for use as a filter. - // To be suitable, it must be null instantiated and - // it's child must not be nullable. We want to filter - // the NULL that are a result of null instantiation, not - // original null values. - // - ItemExpr *cval = new (bindWA->wHeap()) SystemLiteral(1); - cval->bindNode(bindWA); - if (bindWA->errStatus()) return this; - - // Null instantiate the value. - // - ValueId niCval = cval->getValueId().nullInstantiate(bindWA, TRUE); - - // Add it to the RETDesc of the Join. - // - ColRefName cvalName("", bindWA->wHeap()); - antiJoin->getRETDesc()->addColumn(bindWA, cvalName , niCval, USER_COLUMN); - - // Add it to the list of null instantiated outputs. - // - ((Join *)antiJoin)->nullInstantiatedOutput().insert(niCval); - - ItemExpr *nullCheck = niCval.getItemExpr(); + ItemExpr *nullCheck = antiJoin->addNullInstIndicatorVar(bindWA).getItemExpr(); CMPASSERT(nullCheck); @@ -3004,6 +2980,35 @@ RelExpr *Join::bindNode(BindWA *bindWA) return bindSelf(bindWA); } // Join::bindNode() +ValueId Join::addNullInstIndicatorVar(BindWA *bindWA, + ItemExpr *indicatorVal) +{ + // Add an indicator variable that can tell us whether + // a left join found a match in the right child table + // or not. The returned ValueId will have the value 1 + // if a match was found, and NULL if no match was found. + + ItemExpr *cval = indicatorVal; + + if (!cval) + cval = new (bindWA->wHeap()) SystemLiteral(1); + cval = cval->bindNode(bindWA); + if (bindWA->errStatus()) + return NULL_VALUE_ID; + + // Null instantiate the value. + ValueId niCval = cval->getValueId().nullInstantiate(bindWA, TRUE); + + // Add it to the RETDesc of the Join. + ColRefName cvalName("", bindWA->wHeap()); + getRETDesc()->addColumn(bindWA, cvalName , niCval, USER_COLUMN); + + // Add it to the list of null instantiated outputs. + nullInstantiatedOutput().insert(niCval); + + return niCval; +} + //++MV // This function builds the BalueIdMap that is used for translating the required // sort key to the right child sort key and backwards @@ -10812,8 +10817,9 @@ RelExpr* Insert::xformUpsertToEfficientTree(BindWA *bindWA) continue; targetColRef = new(bindWA->wHeap()) ColReference( - new(bindWA->wHeap()) ColRefName( - baseCol->getNAColumn()->getFullColRefName(), bindWA->wHeap())); + new(bindWA->wHeap()) ColRefName( + baseCol->getNAColumn()->getFullColRefName(), + bindWA->wHeap())); if (baseCol->getNAColumn()->isClusteringKey()) @@ -10855,10 +10861,8 @@ RelExpr* Insert::xformUpsertToEfficientTree(BindWA *bindWA) pkeyVals->convertToValueIdList(tablePKeyVals,bindWA,ITM_ITEM_LIST); updateToSelectMap().mapValueIdListDown(tablePKeyVals,sourcePKeyVals); - - Join *lj = new(bindWA->wHeap()) Join(child(0),targetTableScan,REL_LEFT_JOIN,keyPred); - + bindWA->getCurrentScope()->xtnmStack()->createXTNM(); @@ -10867,11 +10871,14 @@ RelExpr* Insert::xformUpsertToEfficientTree(BindWA *bindWA) return NULL; bindWA->getCurrentScope()->xtnmStack()->removeXTNM(); - + ValueId nullInstIndicator( + lj->addNullInstIndicatorVar( + bindWA, + new(bindWA->wHeap()) SystemLiteral( + "U", + CharInfo::ISO88591))); ValueIdSet sequenceFunction ; - ItemExpr *constOne = new (bindWA->wHeap()) ConstValue(1); - //Retrieve all the system and user columns of the left join output ValueIdList ljOutCols = NULL; boundLJ->getRETDesc()->getValueIdList(ljOutCols); @@ -10905,7 +10912,6 @@ RelExpr* Insert::xformUpsertToEfficientTree(BindWA *bindWA) seqNode->selectionPred() += selPredOnLead->getValueId(); seqNode->setChild(0,boundLJ); - RelExpr *boundSeqNode = seqNode->bindNode(bindWA); setChild(0,boundSeqNode); @@ -10928,6 +10934,21 @@ RelExpr* Insert::xformUpsertToEfficientTree(BindWA *bindWA) newNewRecArray.insertAt(i,newRecArrList.at(i)); } newRecExprArray() = newNewRecArray; + + ValueId notCoveredNullInstIndicator; + + notCoveredMap.rewriteValueIdUp(notCoveredNullInstIndicator, + nullInstIndicator); + ItemExpr *nvl = new(bindWA->wHeap()) BuiltinFunction( + ITM_NVL, + bindWA->wHeap(), + 2, + notCoveredNullInstIndicator.getItemExpr(), + new(bindWA->wHeap()) SystemLiteral("I", + CharInfo::ISO88591)); + nvl = nvl->bindNode(bindWA); + setProducedMergeIUDIndicator(nvl->getValueId()); + return topNode; } @@ -11110,7 +11131,7 @@ RelExpr* Insert::xformUpsertToMerge(BindWA *bindWA) if (bindWA->errStatus()) return NULL; // Copy the userSecified and canBeSkipped attribute to mergeUpdateInsertExprArray - ValueIdList mergeInsertExprArray = ((MergeUpdate *)mu)->mergeInsertRecExprArray(); + ValueIdList mergeInsertExprArray = mu->mergeInsertRecExprArray(); for (CollIndex i = 0 ; i < newRecExprArray().entries(); i++) { const Assign *assignExpr = (Assign *)newRecExprArray()[i].getItemExpr(); @@ -11565,6 +11586,26 @@ RelExpr *MergeUpdate::bindNode(BindWA *bindWA) } bindWA->setMergeStatement(TRUE); + + // Create a merge IUD indicator, a CHAR(1) CHARACTER SET ISO88591 + // NOT NULL variable, that can be used by index maintenance and + // other operations to find out what action the WHEN clause + // indicated, insert (I), update (U) or delete (D). This will be + // removed in GenericUpdate::normalizeNode() if nobody asked for + // it. The actual value will be produced by the executor work + // method, there is no expression for it. + if (getProducedMergeIUDIndicator() == NULL_VALUE_ID) + { + ItemExpr *mergeIUDIndicator = new(bindWA->wHeap()) NATypeToItem( + new(bindWA->wHeap()) SQLChar( + 1, FALSE, FALSE, FALSE, FALSE, CharInfo::ISO88591)); + + mergeIUDIndicator = mergeIUDIndicator->bindNode(bindWA); + if (bindWA->errStatus()) + return NULL; + setProducedMergeIUDIndicator(mergeIUDIndicator->getValueId()); + } + RelExpr * boundExpr = Update::bindNode(bindWA); if (bindWA->errStatus()) return NULL; @@ -13672,11 +13713,13 @@ RelExpr *LeafInsert::bindNode(BindWA *bindWA) updateToSelectMap().addMapEntry(assign->getTarget(), assign->getSource()); } + if (getReferencedMergeIUDIndicator() != NULL_VALUE_ID) + bindWA->getCurrentScope()->addOuterRef(getReferencedMergeIUDIndicator()); // RelExpr::bindSelf (in GenericUpdate::bindNode) has done this line, but now // any outer refs discovered in bindNode's in the above loop must be added. // For Index Maintenance, these must be exactly the set of baseColRefs vids // (all the target index cols are from the locally-scoped RETDesc left by - // the GenericUpdate::bindNode). + // the GenericUpdate::bindNode), plus the merge IUD indicator, if used. getGroupAttr()->addCharacteristicInputs(bindWA->getCurrentScope()->getOuterRefs()); // The NATable of getTableName() had been set to INDEX_TABLE so that @@ -13814,6 +13857,8 @@ RelExpr *LeafDelete::bindNode(BindWA *bindWA) // See LeafInsert::bindNode for comments on remainder of this method. + if (getReferencedMergeIUDIndicator() != NULL_VALUE_ID) + bindWA->getCurrentScope()->addOuterRef(getReferencedMergeIUDIndicator()); getGroupAttr()->addCharacteristicInputs(bindWA->getCurrentScope()->getOuterRefs()); getTableName().setSpecialType(ExtendedQualName::NORMAL_TABLE); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1450e836/core/sql/optimizer/ImplRule.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/ImplRule.cpp b/core/sql/optimizer/ImplRule.cpp index 6a0fafc..d446c7d 100644 --- a/core/sql/optimizer/ImplRule.cpp +++ b/core/sql/optimizer/ImplRule.cpp @@ -700,7 +700,8 @@ void copyCommonGenericUpdateFields(GenericUpdate *result, result->setTolerateNonFatalError(bef->getTolerateNonFatalError()); result->setNoCheck(bef->noCheck()); result->setPrecondition(bef->getPrecondition()); - + result->setProducedMergeIUDIndicator(bef->getProducedMergeIUDIndicator()); + result->setReferencedMergeIUDIndicator(bef->getReferencedMergeIUDIndicator()); } void copyCommonUpdateFields(Update *result, @@ -711,6 +712,7 @@ void copyCommonUpdateFields(Update *result, result->mergeInsertRecExpr() = bef->mergeInsertRecExpr(); result->mergeInsertRecExprArray() = bef->mergeInsertRecExprArray(); result->mergeUpdatePred() = bef->mergeUpdatePred(); + } void copyCommonDeleteFields(Delete *result, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1450e836/core/sql/optimizer/Inlining.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/Inlining.cpp b/core/sql/optimizer/Inlining.cpp index c98bf29..e0c6c91 100644 --- a/core/sql/optimizer/Inlining.cpp +++ b/core/sql/optimizer/Inlining.cpp @@ -1841,7 +1841,8 @@ RelExpr *GenericUpdate::createIMTree(BindWA *bindWA, if (imNeeded) if (!imTree) { - imTree = createIMNodes(bindWA, useInternalSyskey, index); + imTree = createIMNodes(bindWA, useInternalSyskey, + index, producedMergeIUDIndicator_); if (getOperatorType() == REL_UNARY_UPDATE || getOperatorType() == REL_UNARY_DELETE) setScanLockForIM(child(0)); @@ -1853,14 +1854,16 @@ RelExpr *GenericUpdate::createIMTree(BindWA *bindWA, if (!bindWA->isTrafLoadPrep()) { imTree = new (bindWA->wHeap()) - Union(imTree, createIMNodes(bindWA, useInternalSyskey, index), + Union(imTree, createIMNodes(bindWA, useInternalSyskey, + index, producedMergeIUDIndicator_), NULL, NULL, REL_UNION, CmpCommon::statementHeap(), TRUE, TRUE); imTree->setBlockStmt(isinBlockStmt()); imTree->getInliningInfo().setFlags(II_isIMUnion); } // not bulk load else { RelExpr * oldIMTree = imTree; - imTree = createIMNodes(bindWA, useInternalSyskey, index); + imTree = createIMNodes(bindWA, useInternalSyskey, index, + producedMergeIUDIndicator_); imTree->setChild(0,oldIMTree); } // is bulk load } @@ -1888,6 +1891,7 @@ static RelExpr *createIMNode(BindWA *bindWA, CorrName &tableCorrName, const CorrName &indexCorrName, IndexDesc *index, + const ValueId &mergeIUDIndicator, NABoolean isIMInsert, NABoolean useInternalSyskey, NABoolean isForUpdate, @@ -1992,6 +1996,29 @@ static RelExpr *createIMNode(BindWA *bindWA, preCond = new (bindWA->wHeap()) UnLogic(ITM_NOT, preCond); } + // If we got an IUD indicator passed in, that means that we have + // an IM tree for update, but the actual operation may be an + // insert or a delete, and therefore we may need to suppress the + // index delete or insert, respectively, with a precondition. + if (mergeIUDIndicator != NULL_VALUE_ID) + { + ItemExpr *iudCond = + new(bindWA->wHeap()) BiRelat( + ITM_NOT_EQUAL, + mergeIUDIndicator.getItemExpr(), + new(bindWA->wHeap()) SystemLiteral( + (isIMInsert ? "D" : "I"), + CharInfo::ISO88591)); + + if (preCond == NULL) + preCond = iudCond; + else + preCond = new (bindWA->wHeap()) BiLogic( + ITM_AND, + iudCond, + preCond); + } + // NULL tableDesc here, like all Insert/Update/Delete ctors in SqlParser, // because the LeafXxx::bindNode will call GenericUpdate::bindNode // which will do the appropriate createTableDesc. @@ -2035,6 +2062,7 @@ static RelExpr *createIMNode(BindWA *bindWA, REL_LEAF_DELETE, preCond, bindWA->wHeap()); + imNode->setReferencedMergeIUDIndicator(mergeIUDIndicator); } // The base table's rowsAffected() will get set in ImplRule.cpp, @@ -2059,7 +2087,8 @@ static RelExpr *createIMNode(BindWA *bindWA, RelExpr *GenericUpdate::createIMNodes(BindWA *bindWA, NABoolean useInternalSyskey, - IndexDesc *index) + IndexDesc *index, + const ValueId &mergeIUDIndicator) { // We call getExtFileSetObj (returns QualifiedName), // NOT getExtFileSetName (returns NAString), @@ -2088,11 +2117,12 @@ RelExpr *GenericUpdate::createIMNodes(BindWA *bindWA, // if (getOperatorType() == REL_UNARY_INSERT || getOperatorType() == REL_UNARY_UPDATE || isEffUpsert) - indexInsert = indexOp = createIMNode(bindWA, - tableCorrName, + indexInsert = indexOp = createIMNode(bindWA, + tableCorrName, indexCorrName, - index, - TRUE, + index, + mergeIUDIndicator, + TRUE, useInternalSyskey, isForUpdate, isMerge(), @@ -2105,9 +2135,11 @@ RelExpr *GenericUpdate::createIMNodes(BindWA *bindWA, getOperatorType() == REL_UNARY_UPDATE || isEffUpsert) - indexDelete = indexOp = createIMNode(bindWA, - tableCorrName, indexCorrName, - index, + indexDelete = indexOp = createIMNode(bindWA, + tableCorrName, + indexCorrName, + index, + mergeIUDIndicator, FALSE, useInternalSyskey, isForUpdate, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1450e836/core/sql/optimizer/NormRelExpr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/NormRelExpr.cpp b/core/sql/optimizer/NormRelExpr.cpp index 90e91fa..dbdb70e 100644 --- a/core/sql/optimizer/NormRelExpr.cpp +++ b/core/sql/optimizer/NormRelExpr.cpp @@ -6864,6 +6864,17 @@ RelExpr * GenericUpdate::normalizeNode(NormWA & normWARef) } } + if (producedMergeIUDIndicator_ != NULL_VALUE_ID) + { + ValueId dummy; + if (NOT getGroupAttr()->getCharacteristicOutputs().referencesTheGivenValue( + producedMergeIUDIndicator_, + dummy)) + // nobody asked for the merge IUD indicator, therefore remove + // it, (e.g. simple table without index maintenance) + producedMergeIUDIndicator_ = NULL_VALUE_ID; + } + return normalizedThis; } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1450e836/core/sql/optimizer/RETDesc.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RETDesc.cpp b/core/sql/optimizer/RETDesc.cpp index a81d231..ee675a3 100644 --- a/core/sql/optimizer/RETDesc.cpp +++ b/core/sql/optimizer/RETDesc.cpp @@ -836,17 +836,39 @@ void RETDesc::print(FILE* ofd, const char* indent, const char* title) const #endif } // RETDesc::print() +static void displayDownHelper(RelExpr *re, RETDesc *prevRD, int level) +{ + RETDesc *thisRD = re->getRETDesc(); + char indent[20]; + + snprintf(indent, sizeof(indent), "Level %4d: ", level); + cout << endl + << endl + << indent + << "====== Operator: " << re->getText().data() + << endl + << flush; + if (thisRD != prevRD && thisRD) + thisRD->print(stdout, indent); + else if (thisRD == NULL) + cout << indent << "++++++ RETDesc is NULL" << endl; + else + cout << indent << "++++++ RETDesc is the same as its parent" << endl; + + + for (int c=0; c<re->getArity(); c++) + { + RelExpr *x = re->child(c); + if (x) + { + cout << indent << "++++++ Child " << c << endl; + displayDownHelper(x, thisRD, level+1); + } + } +} // RETDesc::displayDown + /*static*/ void RETDesc::displayDown(RelExpr *re) { - RETDesc *prevRD = NULL; - while (re) { - cout << (Int32)re->getOperatorType() << " " << flush; - RETDesc *thisRD = re->getRETDesc(); - if (thisRD != prevRD) - thisRD->display(); - else - cout << "RETDesc-is-same" << endl; - prevRD = thisRD; - re = re->child(0); - } + displayDownHelper(re, NULL, 1); } // RETDesc::displayDown + http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1450e836/core/sql/optimizer/RelExpr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelExpr.cpp b/core/sql/optimizer/RelExpr.cpp index 74a20bf..293ed6f 100644 --- a/core/sql/optimizer/RelExpr.cpp +++ b/core/sql/optimizer/RelExpr.cpp @@ -12542,6 +12542,8 @@ Int32 GenericUpdate::getArity() const void GenericUpdate::getPotentialOutputValues(ValueIdSet & outputValues) const { outputValues = potentialOutputs_; + if (producedMergeIUDIndicator_ != NULL_VALUE_ID) + outputValues += producedMergeIUDIndicator_; } const NAString GenericUpdate::getUpdTableNameText() const @@ -12717,7 +12719,9 @@ RelExpr * GenericUpdate::copyTopNode(RelExpr *derivedNode, CollHeap* outHeap) result->preconditionTree_ = preconditionTree_->copyTree(outHeap)->castToItemExpr(); result->setPrecondition(precondition_); result->exprsInDerivedClasses_ = exprsInDerivedClasses_; - + result->producedMergeIUDIndicator_ = producedMergeIUDIndicator_; + result->referencedMergeIUDIndicator_ = referencedMergeIUDIndicator_; + return RelExpr::copyTopNode(result, outHeap); } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1450e836/core/sql/optimizer/RelJoin.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelJoin.h b/core/sql/optimizer/RelJoin.h index 2c66b31..e765ae6 100644 --- a/core/sql/optimizer/RelJoin.h +++ b/core/sql/optimizer/RelJoin.h @@ -287,6 +287,8 @@ public: { return nullInstantiatedForRightJoinOutput_; } const ValueIdList & nullInstantiatedForRightJoinOutput() const { return nullInstantiatedForRightJoinOutput_; } + ValueId addNullInstIndicatorVar(BindWA *bindWA, + ItemExpr *indicatorVal = NULL); //++MV // Used for translating the required sort key to the right http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1450e836/core/sql/optimizer/RelUpdate.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelUpdate.h b/core/sql/optimizer/RelUpdate.h index 36569e3..b253a6c 100644 --- a/core/sql/optimizer/RelUpdate.h +++ b/core/sql/optimizer/RelUpdate.h @@ -53,16 +53,22 @@ // The following are logical operators class GenericUpdate; class Insert; +class LeafInsert; class Update; +class MergeUpdate; class Delete; -class LeafInsert; +class MergeDelete; class LeafDelete; +class HbaseDelete; +class HiveInsert; +class HbaseInsert; +class HBaseBulkLoadPrep; + // The following are physical operators class InsertCursor; - class UpdateCursor; - +class HbaseUpdate; class DeleteCursor; // ----------------------------------------------------------------------- @@ -326,13 +332,26 @@ GenericUpdate(const CorrName &name, RelExpr *bindNode(BindWA *bindWA); // Index Maintenance in the Binder - RelExpr *createIMTree(BindWA *bindWA, UpdateColumns *updatedColumns, NABoolean useInternalSyskey); - RelExpr *createIMNodes(BindWA *bindWA, NABoolean useInternalSyskey, IndexDesc *index); - RelExpr *createUndoTree(BindWA *bindWA, UpdateColumns *updatedColumns, NABoolean useInternalSyskey, NABoolean isImOrRiPresent, NABoolean isOrMvPresent, TriggersTempTable *tempTableObj); - RelExpr *createUndoNodes(BindWA *bindWA, NABoolean useInternalSyskey, IndexDesc *index); + RelExpr *createIMTree(BindWA *bindWA, + UpdateColumns *updatedColumns, + NABoolean useInternalSyskey); + RelExpr *createIMNodes(BindWA *bindWA, + NABoolean useInternalSyskey, + IndexDesc *index, + const ValueId &mergeIUDIndicator); + RelExpr *createUndoTree(BindWA *bindWA, + UpdateColumns *updatedColumns, + NABoolean useInternalSyskey, + NABoolean isImOrRiPresent, + NABoolean isOrMvPresent, + TriggersTempTable *tempTableObj); + RelExpr *createUndoNodes(BindWA *bindWA, + NABoolean useInternalSyskey, + IndexDesc *index); RelExpr *createUndoIUDLog(BindWA *bindWA) ; - RelExpr *createUndoTempTable(TriggersTempTable *tempTableObj,BindWA *bindWA); + RelExpr *createUndoTempTable(TriggersTempTable *tempTableObj, + BindWA *bindWA); void nonvirtual_placeholder_func1(); void nonvirtual_placeholder_func2(); @@ -541,6 +560,14 @@ GenericUpdate(const CorrName &name, inline void setPreconditionTree(ItemExpr *pc) { preconditionTree_ = pc; } inline void setPrecondition(const ValueIdSet pc) { precondition_ = pc; exprsInDerivedClasses_ += precondition_; } + inline const ValueId & getProducedMergeIUDIndicator() const + { return producedMergeIUDIndicator_; } + inline void setProducedMergeIUDIndicator(const ValueId &v) + { producedMergeIUDIndicator_ = v; } + inline const ValueId & getReferencedMergeIUDIndicator() const + { return referencedMergeIUDIndicator_; } + inline void setReferencedMergeIUDIndicator(const ValueId &v) + { referencedMergeIUDIndicator_ = v; } virtual ItemExpr * insertValues() { return NULL;} @@ -934,7 +961,28 @@ private: // LeafDelete (IM). Only insert/delete if TRUE ItemExpr *preconditionTree_; ValueIdSet precondition_; - + + // For upsert or merge we need an indicator whether the + // action for a particular row is an insert, update or + // delete. This ValueId is either NULL_VALUE_ID (meaning + // no merge or upsert) or it is an expression evaluating + // to a CHAR(1) CHARACTER SET ISO88591 with these values: + // I this row is being inserted + // U this row is being updated + // D this row is being deleted + // When we inline index maintenance operations, we generate + // a tree for an update. This indicator can be used at + // runtime to suppress index deletion for inserts and to + // suppress index inserts for deletion. Similar actions + // can be taken for other inlined code like RI constraints, + // triggers or MVs (once supported). + ValueId producedMergeIUDIndicator_; + + // the above variable is used at the producer side, the + // actual MERGE or Upsert statement. However, we also need + // to remember it on the consumer side, like an IM insert + // or delete, this is done here: + ValueId referencedMergeIUDIndicator_; }; @@ -1407,6 +1455,10 @@ public: MergeUpdate(const CorrName &name, TableDesc *tabId, + // This is a problem, we shouldn't have two classes + // that share the same OperatorTypeEnum value, but + // REL_UNARY_UPDATE is also used by the Update class + // above. OperatorTypeEnum otype = REL_UNARY_UPDATE, RelExpr *child = NULL, ItemExpr *setExpr = NULL, @@ -1566,6 +1618,10 @@ public: MergeDelete(const CorrName &name, TableDesc *tabId, + // This is a problem, we shouldn't have two classes + // that share the same OperatorTypeEnum value, but + // REL_UNARY_DELETE is also used by the Delete class + // above. OperatorTypeEnum otype = REL_UNARY_DELETE, RelExpr *child = NULL, ItemExpr *insertCols = NULL, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1450e836/core/sql/regress/executor/EXPECTED015.SB ---------------------------------------------------------------------- diff --git a/core/sql/regress/executor/EXPECTED015.SB b/core/sql/regress/executor/EXPECTED015.SB index e3c1767..c6e8e77 100755 --- a/core/sql/regress/executor/EXPECTED015.SB +++ b/core/sql/regress/executor/EXPECTED015.SB @@ -1655,6 +1655,45 @@ K@ I 13 11 --- 2 row(s) selected. +>>delete from t015t8 where i > 10; + +--- 1 row(s) deleted. +>>cqd TRAF_UPSERT_TO_EFF_TREE 'on'; + +--- SQL operation complete. +>>upsert into t015t8 values (1,2,3),(11,12,13) ; + +--- 2 row(s) inserted. +>>cqd TRAF_UPSERT_TO_EFF_TREE reset; + +--- SQL operation complete. +>>select * from t015t8; + +I J K +----------- ----------- ----------- + + 1 2 3 + 11 12 13 + +--- 2 row(s) selected. +>>select * from table (index_table t015t8i1); + +J@ I +----------- ----------- + + 2 1 + 12 11 + +--- 2 row(s) selected. +>>select * from table (index_table t015t8i2); + +K@ I +----------- ----------- + + 3 1 + 13 11 + +--- 2 row(s) selected. >>MERGE INTO t015t8 ON i = 1 +> WHEN MATCHED THEN UPDATE SET j = 12 +> WHEN NOT MATCHED THEN INSERT VALUES (1,3,4); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1450e836/core/sql/regress/executor/TEST015 ---------------------------------------------------------------------- diff --git a/core/sql/regress/executor/TEST015 b/core/sql/regress/executor/TEST015 index a17660e..5701eec 100755 --- a/core/sql/regress/executor/TEST015 +++ b/core/sql/regress/executor/TEST015 @@ -771,6 +771,13 @@ upsert into t015t8 values (1,2,3),(11,12,13) ; select * from t015t8; select * from table (index_table t015t8i1); select * from table (index_table t015t8i2); +delete from t015t8 where i > 10; +cqd TRAF_UPSERT_TO_EFF_TREE 'on'; +upsert into t015t8 values (1,2,3),(11,12,13) ; +cqd TRAF_UPSERT_TO_EFF_TREE reset; +select * from t015t8; +select * from table (index_table t015t8i1); +select * from table (index_table t015t8i2); MERGE INTO t015t8 ON i = 1 WHEN MATCHED THEN UPDATE SET j = 12 WHEN NOT MATCHED THEN INSERT VALUES (1,3,4);
