Repository: incubator-trafodion Updated Branches: refs/heads/master 9409c8ea0 -> b6f49b677
[TRAFODION-1753] Wrong HASH2 partitioning for salted tables Fix to create the HASH2 partitioning function for salted tables with the number of salt buckets as the number of partitions and with only the salt columns as the partitioning key. Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/2f896f58 Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/2f896f58 Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/2f896f58 Branch: refs/heads/master Commit: 2f896f58f485b89dfc01bdc0ee00c776a26e2562 Parents: 9359d88 Author: Hans Zeller <[email protected]> Authored: Thu Jan 14 17:40:48 2016 +0000 Committer: Hans Zeller <[email protected]> Committed: Thu Jan 14 17:40:48 2016 +0000 ---------------------------------------------------------------------- core/sql/optimizer/NATable.cpp | 73 +++++++++++++++------------- core/sql/sqlcat/desc.h | 1 - core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 18 ++----- 3 files changed, 43 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2f896f58/core/sql/optimizer/NATable.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp index d6d24f8..0b01dc2 100644 --- a/core/sql/optimizer/NATable.cpp +++ b/core/sql/optimizer/NATable.cpp @@ -2193,22 +2193,31 @@ static PartitioningFunction * createHash2PartitioningFunction static -NodeMap* createNodeMapForHbase(desc_struct* desc, const NATable* table, NAMemory* heap) +NodeMap* createNodeMapForHbase(desc_struct* desc, const NATable* table, + int numSaltBuckets, NAMemory* heap) { Int32 partns = 0; + Int32 numRegions = 0; desc_struct* hrk = desc; while ( hrk ) { - partns++; - hrk=hrk->header.next; + numRegions++; + hrk=hrk->header.next; } + if (numSaltBuckets <= 1) + partns = numRegions; + else + partns = numSaltBuckets; + NodeMap* nodeMap = new (heap) NodeMap(heap, partns, NodeMapEntry::ACTIVE, NodeMap::HBASE); // get nodeNames of region servers by making a JNI call // do it only for multiple partition table - if (partns > 1 && (CmpCommon::getDefault(TRAF_ALLOW_ESP_COLOCATION) == DF_ON)) { + // TBD: co-location for tables where # of salt buckets and # regions don't match + if (partns > 1 && (CmpCommon::getDefault(TRAF_ALLOW_ESP_COLOCATION) == DF_ON) && + (numSaltBuckets <= 1 || numSaltBuckets == numRegions)) { ARRAY(const char *) nodeNames(heap, partns); if (table->getRegionsNodeName(partns, nodeNames)) { for (Int32 p=0; p < partns; p++) { @@ -2229,13 +2238,15 @@ NodeMap* createNodeMapForHbase(desc_struct* desc, const NATable* table, NAMemory static PartitioningFunction* -createHash2PartitioningFunctionForHBase(desc_struct* desc, const NATable * table, +createHash2PartitioningFunctionForHBase(desc_struct* desc, + const NATable * table, + int numSaltBuckets, NAMemory* heap) { desc_struct* hrk = desc; - NodeMap* nodeMap = createNodeMapForHbase(desc, table, heap); + NodeMap* nodeMap = createNodeMapForHbase(desc, table, numSaltBuckets, heap); Int32 partns = nodeMap->getNumEntries(); @@ -2634,7 +2645,7 @@ createRangePartitioningFunctionForMultiRegionHBase(Int32 partns, const NAColumnArray & partKeyColArray, NAMemory* heap) { - NodeMap* nodeMap = createNodeMapForHbase(desc, table, heap); + NodeMap* nodeMap = createNodeMapForHbase(desc, table, -1, heap); struct desc_struct* partns_desc = ( table->isHbaseCellTable() || table->isHbaseRowTable()) ? @@ -3842,7 +3853,7 @@ NABoolean createNAFileSets(desc_struct * table_desc /*IN*/, order == ASCENDING); if ( table->isHbaseTable() && - indexColumn->isComputedColumnAlways() ) + indexColumn->isSaltColumn() ) { // examples of the saltClause string: @@ -3855,27 +3866,24 @@ NABoolean createNAFileSets(desc_struct * table_desc /*IN*/, strlen(saltClause), CharInfo::ISO88591); - if ( saltExpr && - saltExpr->getOperatorType() == ITM_HASH2_DISTRIB) { - - // get the # of salted partitions from saltClause - ItemExprList csList(CmpCommon::statementHeap()); - saltExpr->findAll(ITM_CONSTANT, csList, FALSE, FALSE); + CMPASSERT(saltExpr && + saltExpr->getOperatorType() == ITM_HASH2_DISTRIB); - // get #salted partitions from last ConstValue in the list - if ( csList.entries() > 0 ) { - ConstValue* ct = (ConstValue*)csList[csList.entries()-1]; - - if ( ct->canGetExactNumericValue() ) { - numOfSaltedPartitions = ct->getExactNumericValue(); - } + // get the # of salted partitions from saltClause + ItemExprList csList(CmpCommon::statementHeap()); + saltExpr->findAll(ITM_CONSTANT, csList, FALSE, FALSE); + + // get #salted partitions from last ConstValue in the list + if ( csList.entries() > 0 ) { + ConstValue* ct = (ConstValue*)csList[csList.entries()-1]; + + if ( ct->canGetExactNumericValue() ) { + numOfSaltedPartitions = ct->getExactNumericValue(); } } - + // collect all ColReference objects into hbaseSaltColumnList. - CMPASSERT(saltExpr != NULL); saltExpr->findAll(ITM_REFERENCE, hbaseSaltColumnList, FALSE, FALSE); - } if (isTheClusteringKey) @@ -4121,22 +4129,21 @@ NABoolean createNAFileSets(desc_struct * table_desc /*IN*/, Int32 splits = findDescEntries(hbd); // Do Hash2 only if the table is salted orignally - // and the current number of partitions is greater than 1. + // and the current number of HBase regions is greater than 1. if ( doHash2 ) doHash2 = (numOfSaltedPartitions > 0 && splits > 1); - if ( doHash2 && hbd && hbd->header.nodetype == DESC_HBASE_HASH2_REGION_TYPE ) { + if ( hbd ) + if ( doHash2 ) { partFunc = createHash2PartitioningFunctionForHBase( ((table_desc_struct*)table_desc)->hbase_regionkey_desc, table, - heap); + numOfSaltedPartitions, + heap); partitioningKeyColumns = hbaseSaltOnColumns; - - } else - if ((!doHash2 && (hbd && hbd->header.nodetype == DESC_HBASE_HASH2_REGION_TYPE)) - || - (hbd && hbd->header.nodetype == DESC_HBASE_RANGE_REGION_TYPE)) + } + else partFunc = createRangePartitioningFunctionForHBase( ((table_desc_struct*)table_desc)->hbase_regionkey_desc, table, @@ -4144,7 +4151,7 @@ NABoolean createNAFileSets(desc_struct * table_desc /*IN*/, heap); else { - // Range partitioned or single partition table + // no region descriptor, range partitioned or single partition table partFunc = createRangePartitioningFunction( files_desc->body.files_desc.partns_desc, partitioningKeyColumns, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2f896f58/core/sql/sqlcat/desc.h ---------------------------------------------------------------------- diff --git a/core/sql/sqlcat/desc.h b/core/sql/sqlcat/desc.h index 3e98d6c..c4f61dc 100644 --- a/core/sql/sqlcat/desc.h +++ b/core/sql/sqlcat/desc.h @@ -61,7 +61,6 @@ enum desc_nodetype { DESC_VIEW_TYPE, DESC_SCHEMA_LABEL_TYPE, DESC_SEQUENCE_GENERATOR_TYPE, - DESC_HBASE_HASH2_REGION_TYPE, DESC_HBASE_RANGE_REGION_TYPE, DESC_ROUTINE_TYPE, DESC_LIBRARY_TYPE http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2f896f58/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp index eb4efbd..0456728 100644 --- a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp +++ b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp @@ -8619,12 +8619,6 @@ void populateRegionDescForEndKey(char* buf, Int32 len, struct desc_struct* targe target->body.hbase_region_desc.endKeyLen = len; } -void populateRegionDescAsHASH2(char* buf, Int32 len, struct desc_struct* target, NAMemory*) -{ - target->header.nodetype = DESC_HBASE_HASH2_REGION_TYPE; - populateRegionDescForEndKey(buf, len, target); -} - void populateRegionDescAsRANGE(char* buf, Int32 len, struct desc_struct* target, NAMemory*) { target->header.nodetype = DESC_HBASE_RANGE_REGION_TYPE; @@ -9402,15 +9396,9 @@ desc_struct * CmpSeabaseDDL::getSeabaseUserTableDesc(const NAString &catName, ExpHbaseInterface* ehi =CmpSeabaseDDL::allocEHI(); ByteArrayList* bal = ehi->getRegionEndKeys(extNameForHbase); - // Set the header.nodetype to either HASH2 or RANGE based on whether - // the table is salted or not. - if (tableIsSalted && CmpCommon::getDefault(HBASE_HASH2_PARTITIONING) == DF_ON) - ((table_desc_struct*)tableDesc)->hbase_regionkey_desc = - assembleDescs(bal, populateRegionDescAsHASH2, STMTHEAP); - else - if ( CmpCommon::getDefault(HBASE_RANGE_PARTITIONING) == DF_ON ) - ((table_desc_struct*)tableDesc)->hbase_regionkey_desc = - assembleDescs(bal, populateRegionDescAsRANGE, STMTHEAP); + // create a list of region descriptors + ((table_desc_struct*)tableDesc)->hbase_regionkey_desc = + assembleDescs(bal, populateRegionDescAsRANGE, STMTHEAP); delete bal; // if this is base table or index and hbase object doesn't exist, then this object
