HAWQ-258. Remove fast sequence
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ae38cfbd Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ae38cfbd Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ae38cfbd Branch: refs/heads/master Commit: ae38cfbd675ef7c60149d595657d896add5ce864 Parents: 875067d Author: Lirong Jian <[email protected]> Authored: Tue Jan 12 16:26:41 2016 +0800 Committer: Lirong Jian <[email protected]> Committed: Fri Jan 15 16:49:07 2016 +0800 ---------------------------------------------------------------------- src/backend/access/appendonly/Makefile | 2 +- src/backend/access/appendonly/aosegfiles.c | 7 - src/backend/access/appendonly/appendonlyam.c | 790 +------------ .../appendonly/appendonlyblockdirectory.c | 1113 ------------------ src/backend/access/external/hd_work_mgr.c | 2 - src/backend/access/heap/heapam.c | 37 - src/backend/access/index/caql.files | 1 - src/backend/access/index/caqlfilemap.json | 3 - src/backend/access/nbtree/nbtinsert.c | 55 - src/backend/access/parquet/parquetsegfiles.c | 5 - src/backend/bootstrap/bootparse.y | 5 - src/backend/catalog/Makefile | 4 +- src/backend/catalog/aoblkdir.c | 264 ----- src/backend/catalog/caql/Makefile | 1 - src/backend/catalog/core/catcoregen.py | 1 - src/backend/catalog/gp_fastsequence.c | 416 ------- src/backend/catalog/heap.c | 1 - src/backend/catalog/index.c | 40 +- src/backend/catalog/pg_appendonly.c | 4 - src/backend/cdb/cdbappendonlystoragewrite.c | 68 +- src/backend/cdb/cdbllize.c | 1 - src/backend/cdb/cdbpath.c | 1 - src/backend/cdb/cdbpersistentrelfile.c | 47 - src/backend/cdb/cdbplan.c | 15 - src/backend/cdb/cdbquerycontextdispatching.c | 53 - src/backend/cdb/cdbtargeteddispatch.c | 2 - src/backend/commands/analyze.c | 1 - src/backend/commands/cluster.c | 7 - src/backend/commands/explain.c | 17 - src/backend/executor/Makefile | 4 +- src/backend/executor/execAmi.c | 12 +- src/backend/executor/execBitmapAOScan.c | 280 ----- src/backend/executor/execBitmapParquetScan.c | 69 -- src/backend/executor/execBitmapTableScan.c | 14 - src/backend/executor/execMain.c | 1 - src/backend/executor/execProcnode.c | 27 +- src/backend/executor/execUtils.c | 2 - src/backend/executor/nodeBitmapAppendOnlyscan.c | 612 ---------- src/backend/gp_libpq_fe/fe-protocol3.c | 6 - src/backend/nodes/copyfuncs.c | 25 - src/backend/nodes/outfast.c | 14 - src/backend/nodes/outfuncs.c | 3 - src/backend/nodes/print.c | 5 +- src/backend/nodes/readfast.c | 16 - src/backend/optimizer/path/indxpath.c | 3 - src/backend/optimizer/plan/createplan.c | 154 --- src/backend/optimizer/plan/planpartition.c | 1 - src/backend/optimizer/plan/setrefs.c | 35 +- src/backend/optimizer/plan/subselect.c | 5 - src/backend/optimizer/util/pathnode.c | 70 -- src/backend/optimizer/util/walkers.c | 1 - src/backend/tcop/utility.c | 8 - src/backend/utils/misc/guc.c | 22 - src/include/catalog/aoblkdir.h | 45 - src/include/catalog/calico.pl | 3 - src/include/catalog/gp_fastsequence.h | 137 --- src/include/catalog/indexing.h | 3 - src/include/catalog/pg_tidycat.h | 1 - src/include/catalog/pg_type.h | 3 - src/include/cdb/cdbappendonlyam.h | 112 -- src/include/cdb/cdbappendonlyblockdirectory.h | 210 ---- src/include/cdb/cdbappendonlystoragewrite.h | 20 - src/include/cdb/cdbquerycontextdispatching.h | 2 - src/include/executor/executor.h | 16 - src/include/executor/nodeBitmapAppendOnlyscan.h | 56 - src/include/nodes/nodes.h | 2 - src/include/optimizer/pathnode.h | 5 - src/test/regress/expected/insert.out | 5 - src/test/regress/expected/insert_optimizer.out | 5 - src/test/regress/sql/insert.sql | 1 - src/test/unit/mock/mock_info.json | 41 - tools/bin/gppylib/data/2.0.json | 93 -- 72 files changed, 36 insertions(+), 5076 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae38cfbd/src/backend/access/appendonly/Makefile ---------------------------------------------------------------------- diff --git a/src/backend/access/appendonly/Makefile b/src/backend/access/appendonly/Makefile index 2e6282a..805acca 100755 --- a/src/backend/access/appendonly/Makefile +++ b/src/backend/access/appendonly/Makefile @@ -29,7 +29,7 @@ include $(top_builddir)/src/Makefile.global override CPPFLAGS := -I$(top_srcdir)/src/backend/gp_libpq_fe $(CPPFLAGS) override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) -OBJS = appendonlyam.o aosegfiles.o aomd.o appendonlywriter.o appendonlytid.o appendonlyblockdirectory.o +OBJS = appendonlyam.o aosegfiles.o aomd.o appendonlywriter.o appendonlytid.o include $(top_srcdir)/src/backend/common.mk http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae38cfbd/src/backend/access/appendonly/aosegfiles.c ---------------------------------------------------------------------- diff --git a/src/backend/access/appendonly/aosegfiles.c b/src/backend/access/appendonly/aosegfiles.c index da160ce..6da6a21 100644 --- a/src/backend/access/appendonly/aosegfiles.c +++ b/src/backend/access/appendonly/aosegfiles.c @@ -40,7 +40,6 @@ #include "catalog/dependency.h" #include "catalog/indexing.h" #include "catalog/namespace.h" -#include "catalog/gp_fastsequence.h" #include "catalog/aoseg.h" #include "cdb/cdbvars.h" #include "executor/spi.h" @@ -93,15 +92,9 @@ InsertInitialSegnoEntry(AppendOnlyEntry *aoEntry, int segno) int natts = 0; bool *nulls; Datum *values; - ItemPointerData tid; Assert(aoEntry != NULL); - InsertFastSequenceEntry(aoEntry->segrelid, - (int64)segno, - 0, - &tid); - if (segno == 0) { return; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae38cfbd/src/backend/access/appendonly/appendonlyam.c ---------------------------------------------------------------------- diff --git a/src/backend/access/appendonly/appendonlyam.c b/src/backend/access/appendonly/appendonlyam.c index 68031f4..a7e88d2 100644 --- a/src/backend/access/appendonly/appendonlyam.c +++ b/src/backend/access/appendonly/appendonlyam.c @@ -76,7 +76,6 @@ #include "catalog/pg_appendonly.h" #include "catalog/pg_attribute_encoding.h" #include "catalog/namespace.h" -#include "catalog/gp_fastsequence.h" #include "cdb/cdbvars.h" #include "cdb/cdbappendonlyam.h" #include "pgstat.h" @@ -306,46 +305,6 @@ SetNextFileSegForRead(AppendOnlyScanDesc scan) */ if(end_of_split > 0) { - /* Initialize the block directory for inserts if needed. */ - if (scan->buildBlockDirectory) - { - ItemPointerData tid; - - /* - * if building the block directory, we need to make sure the - * sequence starts higher than our highest tuple's rownum. In - * the case of upgraded blocks, the highest tuple will - * have tupCount as its row num for non-upgrade cases, which - * use the sequence, it will be enough to start off the end - * of the sequence; note that this is not ideal -- if we are at - * least curSegInfo->tupcount + 1 then we don't even need to - * update the sequence value. - */ - int64 firstSequence = - GetFastSequences(scan->aoEntry->segrelid, - segno, - 1, - NUM_FAST_SEQUENCES, - &tid); - - - AppendOnlyBlockDirectory_Init_forInsert(scan->blockDirectory, - scan->aoEntry, - scan->appendOnlyMetaDataSnapshot, - NULL, - 0, /* lastSequence */ - scan->aos_rd, - segno, /* segno */ - 1 /* columnGroupNo */); - - Assert(!"need contentid here"); - InsertFastSequenceEntry(scan->aoEntry->segrelid, - segno, - firstSequence, - /*TODO, need change in hawq*/ - &tid); - } - finished_all_splits = false; break; } @@ -448,16 +407,14 @@ errcontext_appendonly_insert_block(AppendOnlyInsertDesc aoInsertDesc) char *relationName = NameStr(aoInsertDesc->aoi_rel->rd_rel->relname); int segmentFileNum = aoInsertDesc->cur_segno; int64 headerOffsetInFile = AppendOnlyStorageWrite_CurrentPosition(&aoInsertDesc->storageWrite); - int64 blockFirstRowNum = aoInsertDesc->blockFirstRowNum; int64 bufferCount = aoInsertDesc->bufferCount; errcontext( "Append-Only table '%s', segment file #%d, block header offset in file = " INT64_FORMAT ", " - "block first row number " INT64_FORMAT ", bufferCount " INT64_FORMAT ")", + ", bufferCount " INT64_FORMAT ")", relationName, segmentFileNum, headerOffsetInFile, - blockFirstRowNum, bufferCount); return 0; @@ -595,8 +552,6 @@ CloseWritableFileSeg(AppendOnlyInsertDesc aoInsertDesc) aoInsertDesc->sendback->eof[0] = fileLen; aoInsertDesc->sendback->uncompressed_eof[0] = fileLen_uncompressed; - aoInsertDesc->sendback->nextFastSequence = aoInsertDesc->lastSequence + aoInsertDesc->numSequences - 1; - /* * Update the AO segment info table with our new eof */ @@ -1222,12 +1177,6 @@ LABEL_START_GETNEXTBLOCK: &scan->executorReadBlock, true)) { - if (scan->buildBlockDirectory) - { - Assert(scan->blockDirectory != NULL); - AppendOnlyBlockDirectory_End_forInsert(scan->blockDirectory); - } - /* done reading the file */ if(scan->toCloseFile){ CloseScannedFileSeg(scan); @@ -1237,28 +1186,18 @@ LABEL_START_GETNEXTBLOCK: return false; } - if (scan->buildBlockDirectory) - { - Assert(scan->blockDirectory != NULL); - AppendOnlyBlockDirectory_InsertEntry( - scan->blockDirectory, 0, - scan->executorReadBlock.blockFirstRowNum, - scan->executorReadBlock.headerOffsetInFile, - scan->executorReadBlock.rowCount); - } - - //skip invalid small content blocks - if(!scan->executorReadBlock.isLarge - && scan->executorReadBlock.executorBlockKind == AoExecutorBlockKind_SingleRow - && scan->executorReadBlock.rowCount==0) - { - //skip current block - AppendOnlyStorageRead_SkipCurrentBlock(&scan->storageRead, true); - goto LABEL_START_GETNEXTBLOCK; - }else{ - AppendOnlyExecutorReadBlock_GetContents( - &scan->executorReadBlock); - } + //skip invalid small content blocks + if(!scan->executorReadBlock.isLarge + && scan->executorReadBlock.executorBlockKind == AoExecutorBlockKind_SingleRow + && scan->executorReadBlock.rowCount==0) + { + //skip current block + AppendOnlyStorageRead_SkipCurrentBlock(&scan->storageRead, true); + goto LABEL_START_GETNEXTBLOCK; + }else{ + AppendOnlyExecutorReadBlock_GetContents( + &scan->executorReadBlock); + } return true; } @@ -1349,11 +1288,6 @@ setupNextWriteBlock(AppendOnlyInsertDesc aoInsertDesc) */ AppendOnlyStorageWrite_PadOutForSplit(&aoInsertDesc->storageWrite, aoInsertDesc->usableBlockSize); - /* Set the firstRowNum for the block */ - aoInsertDesc->blockFirstRowNum = aoInsertDesc->lastSequence + 1; - AppendOnlyStorageWrite_SetFirstRowNum(&aoInsertDesc->storageWrite, - aoInsertDesc->blockFirstRowNum); - if(!aoInsertDesc->shouldCompress) { aoInsertDesc->nonCompressedData = @@ -1484,14 +1418,6 @@ finishWriteBlock(AppendOnlyInsertDesc aoInsertDesc) itemCount); } - /* Insert an entry to the block directory */ - AppendOnlyBlockDirectory_InsertEntry( - &aoInsertDesc->blockDirectory, - 0, - aoInsertDesc->blockFirstRowNum, - AppendOnlyStorageWrite_LastWriteBeginPosition(&aoInsertDesc->storageWrite), - itemCount); - Assert(aoInsertDesc->nonCompressedData == NULL); Assert(!AppendOnlyStorageWrite_IsBufferAllocated(&aoInsertDesc->storageWrite)); } @@ -1613,9 +1539,6 @@ appendonly_beginscan(Relation relation, Snapshot appendOnlyMetaDataSnapshot, int //pgstat_initstats(relation); initscan(scan, key); - scan->buildBlockDirectory = false; - scan->blockDirectory = NULL; - return scan; } @@ -1701,623 +1624,6 @@ appendonly_getnext(AppendOnlyScanDesc scan, ScanDirection direction, TupleTableS return tup; } -static void -closeFetchSegmentFile( - AppendOnlyFetchDesc aoFetchDesc) -{ - Assert(aoFetchDesc->currentSegmentFile.isOpen); - - AppendOnlyStorageRead_CloseFile(&aoFetchDesc->storageRead); - - aoFetchDesc->currentSegmentFile.isOpen = false; -} - -static bool -openFetchSegmentFile( - AppendOnlyFetchDesc aoFetchDesc, - int openSegmentFileNum) -{ - int i; - - FileSegInfo *fsInfo; - int segmentFileNum; - int64 logicalEof; - int32 fileSegNo; - - Assert(!aoFetchDesc->currentSegmentFile.isOpen); - - i = 0; - while (true) - { - if (i >= aoFetchDesc->totalSegfiles) - return false; // Segment file not visible in catalog information. - - fsInfo = aoFetchDesc->segmentFileInfo[i]; - segmentFileNum = fsInfo->segno; - if (openSegmentFileNum == segmentFileNum) - { - logicalEof = (int64)fsInfo->eof; - break; - } - i++; - } - - /* - * Don't try to open a segment file when its EOF is 0, since the file may not - * exist. See MPP-8280. - */ - if (logicalEof == 0) - return false; - - MakeAOSegmentFileName( - aoFetchDesc->relation, - openSegmentFileNum, -1, - &fileSegNo, - aoFetchDesc->segmentFileName); - Assert(strlen(aoFetchDesc->segmentFileName) + 1 <= - aoFetchDesc->segmentFileNameMaxLen); - - // UNDONE: Appropriate to use Try here? - if (!AppendOnlyStorageRead_TryOpenFile( - &aoFetchDesc->storageRead, - aoFetchDesc->segmentFileName, - logicalEof, - -1)) - return false; - - aoFetchDesc->currentSegmentFile.num = openSegmentFileNum; - aoFetchDesc->currentSegmentFile.logicalEof = logicalEof; - - aoFetchDesc->currentSegmentFile.isOpen = true; - - return true; -} - -static bool -fetchNextBlock( - AppendOnlyFetchDesc aoFetchDesc) -{ - AppendOnlyExecutorReadBlock *executorReadBlock = - &aoFetchDesc->executorReadBlock; - - /* - * Try to read next block. - */ - if (!AppendOnlyExecutorReadBlock_GetBlockInfo( - &aoFetchDesc->storageRead, - &aoFetchDesc->executorReadBlock, - true)) - return false; // Hit end of range. - - /* - * Unpack information into member variables. - */ - aoFetchDesc->currentBlock.have = true; - aoFetchDesc->currentBlock.fileOffset = - executorReadBlock->headerOffsetInFile; - aoFetchDesc->currentBlock.overallBlockLen = - AppendOnlyStorageRead_OverallBlockLen( - &aoFetchDesc->storageRead); - aoFetchDesc->currentBlock.firstRowNum = - executorReadBlock->blockFirstRowNum; - aoFetchDesc->currentBlock.lastRowNum = - executorReadBlock->blockFirstRowNum + - executorReadBlock->rowCount - 1; - - aoFetchDesc->currentBlock.isCompressed = - executorReadBlock->isCompressed; - aoFetchDesc->currentBlock.isLargeContent = - executorReadBlock->isLarge; - - aoFetchDesc->currentBlock.gotContents = false; - - return true; -} - -static bool -fetchFromCurrentBlock( - AppendOnlyFetchDesc aoFetchDesc, - int64 rowNum, - TupleTableSlot *slot) -{ - Assert(aoFetchDesc->currentBlock.have); - Assert(rowNum >= aoFetchDesc->currentBlock.firstRowNum); - Assert(rowNum <= aoFetchDesc->currentBlock.lastRowNum); - - if (!aoFetchDesc->currentBlock.gotContents) - { - /* - * Do decompression if necessary and get contents. - */ - AppendOnlyExecutorReadBlock_GetContents( - &aoFetchDesc->executorReadBlock); - - aoFetchDesc->currentBlock.gotContents = true; - } - - return AppendOnlyExecutorReadBlock_FetchTuple( - &aoFetchDesc->executorReadBlock, - rowNum, - /* nkeys */ 0, - /* key */ NULL, - slot); -} - -static void -positionFirstBlockOfRange( - AppendOnlyFetchDesc aoFetchDesc) -{ - AppendOnlyBlockDirectoryEntry_GetBeginRange( - &aoFetchDesc->currentBlock.blockDirectoryEntry, - &aoFetchDesc->scanNextFileOffset, - &aoFetchDesc->scanNextRowNum); -} - -static void -positionLimitToEndOfRange( - AppendOnlyFetchDesc aoFetchDesc) -{ - AppendOnlyBlockDirectoryEntry_GetEndRange( - &aoFetchDesc->currentBlock.blockDirectoryEntry, - &aoFetchDesc->scanAfterFileOffset, - &aoFetchDesc->scanLastRowNum); -} - - -static void -positionSkipCurrentBlock( - AppendOnlyFetchDesc aoFetchDesc) -{ - aoFetchDesc->scanNextFileOffset = - aoFetchDesc->currentBlock.fileOffset + - aoFetchDesc->currentBlock.overallBlockLen; - - aoFetchDesc->scanNextRowNum = aoFetchDesc->currentBlock.lastRowNum + 1; -} - -/* - * Scan through blocks to find row. - * - * If row is not represented in any of the blocks covered by the Block Directory, then the row - * falls into a row gap. The row must have been aborted or deleted and reclaimed. - */ -static bool -scanToFetchTuple( - AppendOnlyFetchDesc aoFetchDesc, - int64 rowNum, - TupleTableSlot *slot) -{ - if (aoFetchDesc->scanNextFileOffset >= - aoFetchDesc->scanAfterFileOffset) - return false; // No more blocks requested for range. - - if (aoFetchDesc->currentSegmentFile.logicalEof == - aoFetchDesc->scanNextFileOffset) - return false; // No more blocks in this file. - - if (aoFetchDesc->currentSegmentFile.logicalEof < - aoFetchDesc->scanNextFileOffset) - return false; // UNDONE: Why does our next scan position go beyond logical EOF? - - /* - * Temporarily restrict our reading to just the range. - */ - AppendOnlyStorageRead_SetTemporaryRange( - &aoFetchDesc->storageRead, - aoFetchDesc->scanNextFileOffset, - aoFetchDesc->scanAfterFileOffset); - AppendOnlyExecutionReadBlock_SetSegmentFileNum( - &aoFetchDesc->executorReadBlock, - aoFetchDesc->currentSegmentFile.num); - AppendOnlyExecutionReadBlock_SetPositionInfo( - &aoFetchDesc->executorReadBlock, - aoFetchDesc->scanNextRowNum); - - aoFetchDesc->skipBlockCount = 0; - while (true) - { - /* - * Fetch block starting at scanNextFileOffset. - */ - if (!fetchNextBlock(aoFetchDesc)) - return false; // No more blocks. - - /* - * Examine new current block header information. - */ - if (rowNum < aoFetchDesc->currentBlock.firstRowNum) - { - /* - * Since we have read a new block, the temporary - * range for the read needs to be adjusted - * accordingly. Otherwise, the underlying bufferedRead - * may stop reading more data because of the - * previously-set smaller temporary range. - */ - int64 beginFileOffset = aoFetchDesc->currentBlock.fileOffset; - int64 afterFileOffset = aoFetchDesc->currentBlock.fileOffset + - aoFetchDesc->currentBlock.overallBlockLen; - - AppendOnlyStorageRead_SetTemporaryRange( - &aoFetchDesc->storageRead, - beginFileOffset, - afterFileOffset); - - return false; // Row fell in gap between blocks. - } - - if (rowNum <= aoFetchDesc->currentBlock.lastRowNum) - return fetchFromCurrentBlock(aoFetchDesc, rowNum, slot); - - /* - * Update information to get next block. - */ - Assert(!aoFetchDesc->currentBlock.gotContents); - - /* MPP-17061: reach the end of range covered by block directory entry */ - if ((aoFetchDesc->currentBlock.fileOffset + - aoFetchDesc->currentBlock.overallBlockLen) >= - aoFetchDesc->scanAfterFileOffset) - { - return false; - } - - AppendOnlyExecutionReadBlock_FinishedScanBlock( - &aoFetchDesc->executorReadBlock); - - AppendOnlyStorageRead_SkipCurrentBlock( - &aoFetchDesc->storageRead,true); - aoFetchDesc->skipBlockCount++; - } -} - - -AppendOnlyFetchDesc -appendonly_fetch_init( - Relation relation, - Snapshot appendOnlyMetaDataSnapshot) -{ - AppendOnlyFetchDesc aoFetchDesc; - AppendOnlyEntry *aoentry; - - AppendOnlyStorageAttributes *attr; - - ValidateAppendOnlyMetaDataSnapshot(&appendOnlyMetaDataSnapshot); - PGFunction *fns = NULL; - - StringInfoData titleBuf; - - /* - * increment relation ref count while scanning relation - * - * This is just to make really sure the relcache entry won't go away while - * the scan has a pointer to it. Caller should be holding the rel open - * anyway, so this is redundant in all normal scenarios... - */ - RelationIncrementReferenceCount(relation); - - /* - * allocate scan descriptor - */ - aoFetchDesc = (AppendOnlyFetchDesc) palloc0(sizeof(AppendOnlyFetchDescData)); - - aoFetchDesc->relation = relation; - aoFetchDesc->appendOnlyMetaDataSnapshot = appendOnlyMetaDataSnapshot; - - aoFetchDesc->initContext = CurrentMemoryContext; - - aoFetchDesc->segmentFileNameMaxLen = AOSegmentFilePathNameLen(relation) + 1; - aoFetchDesc->segmentFileName = - (char*)palloc(aoFetchDesc->segmentFileNameMaxLen); - aoFetchDesc->segmentFileName[0] = '\0'; - - initStringInfo(&titleBuf); - appendStringInfo(&titleBuf, "Fetch of Append-Only Row-Oriented relation '%s'", - RelationGetRelationName(relation)); - aoFetchDesc->title = titleBuf.data; - - /* - * Get the pg_appendonly information for this table - */ - aoentry = GetAppendOnlyEntry(RelationGetRelid(relation), appendOnlyMetaDataSnapshot); - - aoFetchDesc->aoEntry = aoentry; - - /* - * Fill in Append-Only Storage layer attributes. - */ - attr = &aoFetchDesc->storageAttributes; - - /* - * These attributes describe the AppendOnly format to be scanned. - */ - if (aoentry->compresstype == NULL || pg_strcasecmp(aoentry->compresstype, "none") == 0) - attr->compress = false; - else - attr->compress = true; - if (aoentry->compresstype != NULL) - attr->compressType = aoentry->compresstype; - else - attr->compressType = "none"; - attr->compressLevel = aoentry->compresslevel; - attr->checksum = aoentry->checksum; - attr->safeFSWriteSize = aoentry->safefswritesize; - attr->splitsize = aoentry->splitsize; - attr->version = aoentry->version; - - AORelationVersion_CheckValid(attr->version); - - aoFetchDesc->usableBlockSize = aoentry->blocksize; - /* AppendOnlyStorage_GetUsableBlockSize(aoentry->blocksize); */ - - /* - * Get information about all the file segments we need to scan - * Currently, fetch operation is disabled. So we just set the - * segmentFileInfo NULL. - */ - aoFetchDesc->segmentFileInfo = NULL; - /* - GetAllFileSegInfo( - relation, - aoentry, - appendOnlyMetaDataSnapshot, - false, - &aoFetchDesc->totalSegfiles); - */ - AppendOnlyStorageRead_Init( - &aoFetchDesc->storageRead, - aoFetchDesc->initContext, - aoFetchDesc->usableBlockSize, - NameStr(aoFetchDesc->relation->rd_rel->relname), - aoFetchDesc->title, - &aoFetchDesc->storageAttributes); - - - fns = RelationGetRelationCompressionFuncs(relation); - aoFetchDesc->storageRead.compression_functions = fns; - - if (fns) - { - PGFunction cons = fns[COMPRESSION_CONSTRUCTOR]; - CompressionState *cs; - StorageAttributes sa; - - sa.comptype = aoentry->compresstype; - sa.complevel = aoentry->compresslevel; - sa.blocksize = aoentry->blocksize; - - - cs = callCompressionConstructor(cons, RelationGetDescr(relation), - &sa, - false /* decompress */); - aoFetchDesc->storageRead.compressionState = cs; - } - - AppendOnlyExecutorReadBlock_Init( - &aoFetchDesc->executorReadBlock, - aoFetchDesc->relation, - aoFetchDesc->initContext, - &aoFetchDesc->storageRead, - aoFetchDesc->usableBlockSize); - - AppendOnlyBlockDirectory_Init_forSearch( - &aoFetchDesc->blockDirectory, - aoentry, - appendOnlyMetaDataSnapshot, - aoFetchDesc->segmentFileInfo, - aoFetchDesc->totalSegfiles, - aoFetchDesc->relation, - 1); - - return aoFetchDesc; - -} - -/* - * appendonly_fetch -- fetch the tuple for a given tid. - * - * If the 'slot' is not NULL, the fetched tuple will be assigned to the slot. - * - * Return true if such a tuple is found. Otherwise, return false. - */ -bool -appendonly_fetch( - AppendOnlyFetchDesc aoFetchDesc, - AOTupleId *aoTupleId, - TupleTableSlot *slot) -{ - int segmentFileNum = AOTupleIdGet_segmentFileNum(aoTupleId); - int64 rowNum = AOTupleIdGet_rowNum(aoTupleId); - - /* - * Do we have a current block? If it has the requested tuple, - * that would be a great performance optimization. - */ - if (aoFetchDesc->currentBlock.have) - { - if (segmentFileNum == aoFetchDesc->currentSegmentFile.num && - segmentFileNum == aoFetchDesc->blockDirectory.currentSegmentFileNum) - { - if (rowNum >= aoFetchDesc->currentBlock.firstRowNum && - rowNum <= aoFetchDesc->currentBlock.lastRowNum) - return fetchFromCurrentBlock(aoFetchDesc, rowNum, slot); - - /* - * Otherwize, if the current Block Directory entry covers the request tuples, - * lets use its information as another performance optimization. - */ - if (AppendOnlyBlockDirectoryEntry_RangeHasRow( - &aoFetchDesc->currentBlock.blockDirectoryEntry, - rowNum)) - { - /* - * The tuple is covered by the current Block Directory entry, but is it - * before or after our current block? - */ - if (rowNum < aoFetchDesc->currentBlock.firstRowNum) - { - /* - * XXX This could happen when an insert is cancelled. In that case, we - * fetched the next block that has a higher firstRowNum when we - * try to find the first cancelled row. So for the second or any - * cancelled row, we enter here, and re-read the previous block. - * This seems inefficient. - * - * We may be able to fix this by adding an entry to the block - * directory for those cancelled inserts. - */ - - /* - * Set scan range to prior blocks. - */ - positionFirstBlockOfRange(aoFetchDesc); - - // Set limit to before current block. - aoFetchDesc->scanAfterFileOffset = - aoFetchDesc->currentBlock.fileOffset; - - aoFetchDesc->scanLastRowNum = - aoFetchDesc->currentBlock.firstRowNum - 1; - } - else - { - /* - * Set scan range to following blocks. - */ - positionSkipCurrentBlock(aoFetchDesc); - - positionLimitToEndOfRange(aoFetchDesc); - } - - if (scanToFetchTuple(aoFetchDesc, rowNum, slot)) - return true; - - if (slot != NULL) - ExecClearTuple(slot); - return false; // Segment file not in aoseg table.. - } - } - } - -// resetCurrentBlockInfo(aoFetchDesc); - - /* - * Open or switch open, if necessary. - */ - if (aoFetchDesc->currentSegmentFile.isOpen && - segmentFileNum != aoFetchDesc->currentSegmentFile.num) - { -#ifdef USE_ASSERT_CHECKING - if (segmentFileNum < aoFetchDesc->currentSegmentFile.num) - ereport(WARNING, - (errmsg("Append-only fetch requires scan prior segment file: " - "segmentFileNum %d, rowNum " INT64_FORMAT - ", currentSegmentFileNum %d", - segmentFileNum, rowNum, aoFetchDesc->currentSegmentFile.num))); -#endif - closeFetchSegmentFile(aoFetchDesc); - - Assert(!aoFetchDesc->currentSegmentFile.isOpen); - } - - if (!aoFetchDesc->currentSegmentFile.isOpen) - { - if (!openFetchSegmentFile( - aoFetchDesc, - segmentFileNum)) - { - if (slot != NULL) - ExecClearTuple(slot); - return false; // Segment file not in aoseg table.. - // Must be aborted or deleted and reclaimed. - } - } - - /* - * Need to get the Block Directory entry that covers the TID. - */ - if (!AppendOnlyBlockDirectory_GetEntry( - &aoFetchDesc->blockDirectory, - aoTupleId, - 0, - &aoFetchDesc->currentBlock.blockDirectoryEntry)) - { - if (slot != NULL) - { - ExecClearTuple(slot); - } - return false; /* Row not represented in Block Directory. */ - /* Must be aborted or deleted and reclaimed. */ - } - - /* - * Set scan range covered by new Block Directory entry. - */ - positionFirstBlockOfRange(aoFetchDesc); - - positionLimitToEndOfRange(aoFetchDesc); - - if (scanToFetchTuple(aoFetchDesc, rowNum, slot)) - return true; - - if (slot != NULL) - ExecClearTuple(slot); - return false; // Segment file not in aoseg table.. -} - -void -appendonly_fetch_detail( - AppendOnlyFetchDesc aoFetchDesc, - AppendOnlyFetchDetail *aoFetchDetail) -{ - aoFetchDetail->rangeFileOffset = - aoFetchDesc->currentBlock.blockDirectoryEntry.range.fileOffset; - aoFetchDetail->rangeFirstRowNum = - aoFetchDesc->currentBlock.blockDirectoryEntry.range.firstRowNum; - aoFetchDetail->rangeAfterFileOffset = - aoFetchDesc->currentBlock.blockDirectoryEntry.range.afterFileOffset; - aoFetchDetail->rangeLastRowNum = - aoFetchDesc->currentBlock.blockDirectoryEntry.range.lastRowNum; - - aoFetchDetail->skipBlockCount = aoFetchDesc->skipBlockCount; - - aoFetchDetail->blockFileOffset = aoFetchDesc->currentBlock.fileOffset; - aoFetchDetail->blockOverallLen = aoFetchDesc->currentBlock.overallBlockLen; - aoFetchDetail->blockFirstRowNum = aoFetchDesc->currentBlock.firstRowNum; - aoFetchDetail->blockLastRowNum = aoFetchDesc->currentBlock.lastRowNum; - aoFetchDetail->isCompressed = aoFetchDesc->currentBlock.isCompressed; - aoFetchDetail->isLargeContent = aoFetchDesc->currentBlock.isLargeContent; -} - -void -appendonly_fetch_finish(AppendOnlyFetchDesc aoFetchDesc) -{ - RelationDecrementReferenceCount(aoFetchDesc->relation); - - AppendOnlyStorageRead_CloseFile(&aoFetchDesc->storageRead); - - AppendOnlyStorageRead_FinishSession(&aoFetchDesc->storageRead); - - AppendOnlyExecutorReadBlock_Finish(&aoFetchDesc->executorReadBlock); - - AppendOnlyBlockDirectory_End_forSearch(&aoFetchDesc->blockDirectory); - - if (aoFetchDesc->segmentFileInfo) - { - FreeAllSegFileInfo(aoFetchDesc->segmentFileInfo, aoFetchDesc->totalSegfiles); - pfree(aoFetchDesc->segmentFileInfo); - aoFetchDesc->segmentFileInfo = NULL; - } - - pfree(aoFetchDesc->aoEntry); - aoFetchDesc->aoEntry = NULL; - - pfree(aoFetchDesc->segmentFileName); - aoFetchDesc->segmentFileName = NULL; - - pfree(aoFetchDesc->title); -} - /* * appendonly_insert_init * @@ -2334,7 +1640,6 @@ appendonly_insert_init(Relation rel, ResultRelSegFileInfo *segfileinfo) AppendOnlyInsertDesc aoInsertDesc; AppendOnlyEntry *aoentry; int maxtupsize; - int64 firstSequence = 0; PGFunction *fns; int desiredOverflowBytes = 0; size_t (*desiredCompressionSize)(size_t input); @@ -2366,7 +1671,6 @@ appendonly_insert_init(Relation rel, ResultRelSegFileInfo *segfileinfo) aoInsertDesc->appendFilePathName[0] = '\0'; aoInsertDesc->bufferCount = 0; - aoInsertDesc->blockFirstRowNum = 0; aoInsertDesc->insertCount = 0; aoInsertDesc->varblockCount = 0; aoInsertDesc->rowCount = 0; @@ -2472,13 +1776,6 @@ appendonly_insert_init(Relation rel, ResultRelSegFileInfo *segfileinfo) (aoentry->compresstype ? aoentry->compresstype : "<none>"), attr->compressLevel); - /* - * Temporarily set the firstRowNum for the block so that we can - * calculate the correct header length. - */ - AppendOnlyStorageWrite_SetFirstRowNum(&aoInsertDesc->storageWrite, - 1); - aoInsertDesc->completeHeaderLen = AppendOnlyStorageWrite_CompleteHeaderLen( &aoInsertDesc->storageWrite, @@ -2509,31 +1806,8 @@ appendonly_insert_init(Relation rel, ResultRelSegFileInfo *segfileinfo) Assert(!ItemPointerIsValid(&aoInsertDesc->fsInfo->sequence_tid)); Assert(aoInsertDesc->fsInfo->segno == segfileinfo->segno); - /* - firstSequence = - GetFastSequences(aoInsertDesc->aoEntry->segrelid, - segfileinfo->segno, - aoInsertDesc->rowCount + 1, - NUM_FAST_SEQUENCES, - &aoInsertDesc->fsInfo->sequence_tid); - */ - firstSequence = aoInsertDesc->rowCount + 1; - aoInsertDesc->numSequences = NUM_FAST_SEQUENCES; - - /* Set last_sequence value */ - Assert(firstSequence > aoInsertDesc->rowCount); - aoInsertDesc->lastSequence = firstSequence - 1; - setupNextWriteBlock(aoInsertDesc); - /* Initialize the block directory. */ - AppendOnlyBlockDirectory_Init_forInsert( - &(aoInsertDesc->blockDirectory), - aoentry, - aoInsertDesc->appendOnlyMetaDataSnapshot, // CONCERN: Safe to assume all block directory entries for segment are "covered" by same exclusive lock. - aoInsertDesc->fsInfo, aoInsertDesc->lastSequence, - rel, segfileinfo->segno, 1); - return aoInsertDesc; } @@ -2766,48 +2040,12 @@ appendonly_insert_init(Relation rel, ResultRelSegFileInfo *segfileinfo) } aoInsertDesc->insertCount++; - aoInsertDesc->lastSequence++; - if (aoInsertDesc->numSequences > 0) - (aoInsertDesc->numSequences)--; - - Assert(aoInsertDesc->numSequences >= 0); - pgstat_count_heap_insert(relation); *tupleOid = MemTupleGetOid(tup, aoInsertDesc->mt_bind); AOTupleIdInit_Init(aoTupleId); AOTupleIdInit_segmentFileNum(aoTupleId, aoInsertDesc->cur_segno); - AOTupleIdInit_rowNum(aoTupleId, aoInsertDesc->lastSequence); - - /* - * If the allocated fast sequence numbers are used up, we request for - * a next list of fast sequence numbers. - */ - if (aoInsertDesc->numSequences == 0) - { - int64 firstSequence; - - /* - * in hawq, catalog are in memory heap table, - * ItemPointer of tuple is invalid. - */ - if (Gp_role == GP_ROLE_EXECUTE) - { - /* - firstSequence = GetFastSequences(aoInsertDesc->aoEntry->segrelid, - aoInsertDesc->cur_segno, aoInsertDesc->lastSequence + 1, - NUM_FAST_SEQUENCES, &aoInsertDesc->fsInfo->sequence_tid); - */ - firstSequence = aoInsertDesc->lastSequence + 1; - } else { - firstSequence = GetFastSequencesByTid( - &aoInsertDesc->fsInfo->sequence_tid, - aoInsertDesc->lastSequence + 1, NUM_FAST_SEQUENCES); - } - Assert(firstSequence == aoInsertDesc->lastSequence + 1); - aoInsertDesc->numSequences = NUM_FAST_SEQUENCES; - } if (Debug_appendonly_print_insert_tuple) { @@ -2840,8 +2078,6 @@ appendonly_insert_finish(AppendOnlyInsertDesc aoInsertDesc) CloseWritableFileSeg(aoInsertDesc); - AppendOnlyBlockDirectory_End_forInsert(&(aoInsertDesc->blockDirectory)); - AppendOnlyStorageWrite_FinishSession(&aoInsertDesc->storageWrite); pfree(aoInsertDesc->aoEntry); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae38cfbd/src/backend/access/appendonly/appendonlyblockdirectory.c ---------------------------------------------------------------------- diff --git a/src/backend/access/appendonly/appendonlyblockdirectory.c b/src/backend/access/appendonly/appendonlyblockdirectory.c deleted file mode 100644 index df5b45d..0000000 --- a/src/backend/access/appendonly/appendonlyblockdirectory.c +++ /dev/null @@ -1,1113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/*----------------------------------------------------------------------------- - * - * appendonlyblockdirectory - * maintain the block directory to blocks in an append-only relation - * files. - * - *----------------------------------------------------------- - */ -#include "postgres.h" - -#include "cdb/cdbappendonlyblockdirectory.h" -#include "catalog/aoblkdir.h" -#include "access/heapam.h" -#include "access/genam.h" -#include "catalog/indexing.h" -#include "parser/parse_oper.h" -#include "utils/lsyscache.h" -#include "utils/memutils.h" -#include "utils/guc.h" -#include "cdb/cdbappendonlyam.h" - -int gp_blockdirectory_entry_min_range = 0; -int gp_blockdirectory_minipage_size = NUM_MINIPAGE_ENTRIES; - -static inline uint32 minipage_size(uint32 nEntry) -{ - return offsetof(Minipage, entry) + - sizeof(MinipageEntry) * nEntry; -} - -static void load_last_minipage( - AppendOnlyBlockDirectory *blockDirectory, - int64 lastSequence, - int columnGroupNo); -static void init_scankeys( - TupleDesc tupleDesc, - int nkeys, ScanKey scanKeys, - StrategyNumber *strategyNumbers); -static int find_minipage_entry( - Minipage *minipage, - uint32 numEntries, - int64 rowNum); -static void extract_minipage( - AppendOnlyBlockDirectory *blockDirectory, - HeapTuple tuple, - TupleDesc tupleDesc, - int columnGroupNo); -static void write_minipage(AppendOnlyBlockDirectory *blockDirectory, - int columnGroupNo); - -void -AppendOnlyBlockDirectoryEntry_GetBeginRange( - AppendOnlyBlockDirectoryEntry *directoryEntry, - int64 *fileOffset, - int64 *firstRowNum) -{ - *fileOffset = directoryEntry->range.fileOffset; - *firstRowNum = directoryEntry->range.firstRowNum; -} - -void -AppendOnlyBlockDirectoryEntry_GetEndRange( - AppendOnlyBlockDirectoryEntry *directoryEntry, - int64 *afterFileOffset, - int64 *lastRowNum) -{ - *afterFileOffset = directoryEntry->range.afterFileOffset; - *lastRowNum = directoryEntry->range.lastRowNum; -} - -bool -AppendOnlyBlockDirectoryEntry_RangeHasRow( - AppendOnlyBlockDirectoryEntry *directoryEntry, - int64 checkRowNum) -{ - return (checkRowNum >= directoryEntry->range.firstRowNum && - checkRowNum <= directoryEntry->range.lastRowNum); -} - -/* - * init_internal - * - * Initialize the block directory structure. - */ -static void -init_internal(AppendOnlyBlockDirectory *blockDirectory) -{ - MemoryContext oldcxt; - int numScanKeys; - TupleDesc heapTupleDesc; - TupleDesc idxTupleDesc; - int groupNo; - - Assert(blockDirectory->blkdirRel != NULL); - Assert(blockDirectory->blkdirIdx != NULL); - - blockDirectory->memoryContext = - AllocSetContextCreate(CurrentMemoryContext, - "BlockDirectoryContext", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); - - oldcxt = MemoryContextSwitchTo(blockDirectory->memoryContext); - - heapTupleDesc = RelationGetDescr(blockDirectory->blkdirRel); - blockDirectory->values = palloc0(sizeof(Datum) * heapTupleDesc->natts); - blockDirectory->nulls = palloc0(sizeof(bool) * heapTupleDesc->natts); - blockDirectory->numScanKeys = 3; - numScanKeys = blockDirectory->numScanKeys; - blockDirectory->scanKeys = palloc0(numScanKeys * sizeof(ScanKeyData)); - - blockDirectory->strategyNumbers = palloc0(numScanKeys * sizeof(StrategyNumber)); - blockDirectory->strategyNumbers[0] = BTEqualStrategyNumber; - blockDirectory->strategyNumbers[1] = BTEqualStrategyNumber; - blockDirectory->strategyNumbers[2] = BTLessEqualStrategyNumber; - - idxTupleDesc = RelationGetDescr(blockDirectory->blkdirIdx); - - init_scankeys(idxTupleDesc, numScanKeys, - blockDirectory->scanKeys, - blockDirectory->strategyNumbers); - - /* Initialize the last minipage */ - blockDirectory->minipages = - palloc0(sizeof(MinipagePerColumnGroup) * blockDirectory->numColumnGroups); - for (groupNo = 0; groupNo < blockDirectory->numColumnGroups; groupNo++) - { - MinipagePerColumnGroup *minipageInfo = &blockDirectory->minipages[groupNo]; - - minipageInfo->minipage = - palloc0(minipage_size(NUM_MINIPAGE_ENTRIES)); - minipageInfo->numMinipageEntries = 0; - } - - - MemoryContextSwitchTo(oldcxt); -} - -/* - * AppendOnlyBlockDirectory_Init_forSearch - * - * Initialize the block directory to handle the lookup. - * - * If the block directory relation for this appendonly relation - * does not exist before calling this function, set blkdirRel - * and blkdirIdx to NULL, and return. - */ -void -AppendOnlyBlockDirectory_Init_forSearch( - AppendOnlyBlockDirectory *blockDirectory, - AppendOnlyEntry *aoEntry, - Snapshot appendOnlyMetaDataSnapshot, - FileSegInfo **segmentFileInfo, - int totalSegfiles, - Relation aoRel, - int numColumnGroups) -{ - Assert(aoEntry != NULL); - - blockDirectory->aoRel = aoRel; - - if (!OidIsValid(aoEntry->blkdirrelid)) - { - Assert(!OidIsValid(aoEntry->blkdiridxid)); - blockDirectory->blkdirRel = NULL; - blockDirectory->blkdirIdx = NULL; - - return; - } - - if (Debug_appendonly_print_blockdirectory) - ereport(LOG, - (errmsg("Append-only block directory init for search: " - "(totalSegfiles, numColumnGroups)=" - "(%d, %d)", - totalSegfiles, numColumnGroups))); - - blockDirectory->segmentFileInfo = segmentFileInfo; - blockDirectory->totalSegfiles = totalSegfiles; - blockDirectory->aoRel = aoRel; - blockDirectory->appendOnlyMetaDataSnapshot = appendOnlyMetaDataSnapshot; - blockDirectory->numColumnGroups = numColumnGroups; - blockDirectory->currentSegmentFileNum = -1; - - Assert(OidIsValid(aoEntry->blkdirrelid)); - - blockDirectory->blkdirRel = - heap_open(aoEntry->blkdirrelid, AccessShareLock); - - Assert(OidIsValid(aoEntry->blkdiridxid)); - - blockDirectory->blkdirIdx = - index_open(aoEntry->blkdiridxid, AccessShareLock); - - init_internal(blockDirectory); -} - -/* - * AppendOnlyBlockDirectory_Init_forInsert - * - * Initialize the block directory to handle the inserts. - * - * If the block directory relation for this appendonly relation - * does not exist before calling this function, set blkdirRel - * and blkdirIdx to NULL, and return. - */ -void -AppendOnlyBlockDirectory_Init_forInsert( - AppendOnlyBlockDirectory *blockDirectory, - AppendOnlyEntry *aoEntry, - Snapshot appendOnlyMetaDataSnapshot, - FileSegInfo *segmentFileInfo, - int64 lastSequence, - Relation aoRel, - int segno, - int numColumnGroups) -{ - int groupNo; - - Assert(aoEntry != NULL); - - blockDirectory->aoRel = aoRel; - blockDirectory->appendOnlyMetaDataSnapshot = appendOnlyMetaDataSnapshot; - - if (!OidIsValid(aoEntry->blkdirrelid)) - { - Assert(!OidIsValid(aoEntry->blkdiridxid)); - blockDirectory->blkdirRel = NULL; - blockDirectory->blkdirIdx = NULL; - - return; - } - - blockDirectory->segmentFileInfo = NULL; - blockDirectory->totalSegfiles = -1; - blockDirectory->currentSegmentFileInfo = segmentFileInfo; - - blockDirectory->currentSegmentFileNum = segno; - blockDirectory->numColumnGroups = numColumnGroups; - - Assert(OidIsValid(aoEntry->blkdirrelid)); - - blockDirectory->blkdirRel = - heap_open(aoEntry->blkdirrelid, RowExclusiveLock); - - Assert(OidIsValid(aoEntry->blkdiridxid)); - - blockDirectory->blkdirIdx = - index_open(aoEntry->blkdiridxid, RowExclusiveLock); - - init_internal(blockDirectory); - - if (Debug_appendonly_print_blockdirectory) - ereport(LOG, - (errmsg("Append-only block directory init for insert: " - "(segno, numColumnGroups, isAOCol, lastSequence)=" - "(%d, %d, " INT64_FORMAT ")", - segno, numColumnGroups, lastSequence))); - - /* - * Load the last minipages from the block directory relation. - */ - for(groupNo = 0; groupNo < blockDirectory->numColumnGroups; groupNo++) - { - load_last_minipage(blockDirectory, lastSequence, groupNo); - } -} - -static bool -set_directoryentry_range( - AppendOnlyBlockDirectory *blockDirectory, - int columnGroupNo, - int entry_no, - AppendOnlyBlockDirectoryEntry *directoryEntry) -{ - MinipagePerColumnGroup *minipageInfo = - &blockDirectory->minipages[columnGroupNo]; - FileSegInfo *fsInfo; - MinipageEntry *entry; - MinipageEntry *next_entry = NULL; - - Assert(entry_no >= 0 && ((uint32)entry_no) < minipageInfo->numMinipageEntries); - - fsInfo = blockDirectory->currentSegmentFileInfo; - Assert(fsInfo != NULL); - - entry = &(minipageInfo->minipage->entry[entry_no]); - if (((uint32)entry_no) < minipageInfo->numMinipageEntries - 1) - { - next_entry = &(minipageInfo->minipage->entry[entry_no + 1]); - } - - directoryEntry->range.fileOffset = entry->fileOffset; - directoryEntry->range.firstRowNum = entry->firstRowNum; - if (next_entry != NULL) - { - directoryEntry->range.afterFileOffset = next_entry->fileOffset; - } - else - { - directoryEntry->range.afterFileOffset = fsInfo->eof; - } - - directoryEntry->range.lastRowNum = entry->firstRowNum + entry->rowCount - 1; - if (next_entry == NULL && gp_blockdirectory_entry_min_range != 0) - { - directoryEntry->range.lastRowNum = (~(((int64)1) << 63)); /* set to the maximal value */ - } - - /* - * When crashes during inserts, or cancellation during inserts, - * the block directory may contain out-of-date entries. We - * check for the end of file here. If the requested directory entry - * is after the end of file, return false. - */ - if (directoryEntry->range.fileOffset > fsInfo->eof) - return false; - - if ((directoryEntry->range.afterFileOffset > fsInfo->eof)) - { - directoryEntry->range.afterFileOffset = fsInfo->eof; - } - - if (Debug_appendonly_print_blockdirectory) - ereport(LOG, - (errmsg("Append-only block directory find entry: " - "(columnGroupNo, firstRowNum, fileOffset, lastRowNum, afterFileOffset) = " - "(%d, " INT64_FORMAT ", " INT64_FORMAT ", " INT64_FORMAT ", " INT64_FORMAT ")", - columnGroupNo, directoryEntry->range.firstRowNum, - directoryEntry->range.fileOffset, directoryEntry->range.lastRowNum, - directoryEntry->range.afterFileOffset))); - - return true; -} - -/* - * AppendOnlyBlockDirectory_GetEntry - * - * Find a directory entry for the given AOTupleId in the block directory. - * If such an entry is found, return true. Otherwise, return false. - * - * The range for directoryEntry is assigned accordingly in this function. - * - * The block directory for the appendonly table should exist before calling - * this function. - */ -bool -AppendOnlyBlockDirectory_GetEntry( - AppendOnlyBlockDirectory *blockDirectory, - AOTupleId *aoTupleId, - int columnGroupNo, - AppendOnlyBlockDirectoryEntry *directoryEntry) -{ - int segmentFileNum = AOTupleIdGet_segmentFileNum(aoTupleId); - int64 rowNum = AOTupleIdGet_rowNum(aoTupleId); - int i; - Relation blkdirRel = blockDirectory->blkdirRel; - Relation blkdirIdx = blockDirectory->blkdirIdx; - int numScanKeys = blockDirectory->numScanKeys; - ScanKey scanKeys = blockDirectory->scanKeys; - - TupleDesc heapTupleDesc; - TupleDesc idxTupleDesc; - FileSegInfo *fsInfo = NULL; - IndexScanDesc idxScanDesc; - HeapTuple tuple = NULL; - MinipagePerColumnGroup *minipageInfo = - &blockDirectory->minipages[columnGroupNo]; - int entry_no = -1; - int tmpGroupNo; - - if (blkdirRel == NULL || blkdirIdx == NULL) - { - Assert(RelationIsValid(blockDirectory->aoRel)); - - ereport(ERROR, - (errcode(ERRCODE_GP_INTERNAL_ERROR), - errmsg("Block directory for append-only relation '%s' does not exist", - RelationGetRelationName(blockDirectory->aoRel)))); - return false; - } - - if (Debug_appendonly_print_blockdirectory) - ereport(LOG, - (errmsg("Append-only block directory get entry: " - "(columnGroupNo, segmentFileNum, rowNum) = " - "(%d, %d, " INT64_FORMAT ")", - columnGroupNo, segmentFileNum, rowNum))); - - /* - * If the segment file number is the same as - * blockDirectory->currentSegmentFileNum, the in-memory minipage - * may contain such an entry. We search the in-memory minipage - * first. If such an entry can not be found, we search for the - * appropriate minipage by using the block directory btree index. - */ - if (segmentFileNum == blockDirectory->currentSegmentFileNum && - minipageInfo->numMinipageEntries > 0) - { - Assert(blockDirectory->currentSegmentFileInfo != NULL); - - /* - * Check if the existing minipage contains the requested - * rowNum. If so, just get it. - */ - entry_no = find_minipage_entry(minipageInfo->minipage, - minipageInfo->numMinipageEntries, - rowNum); - if (entry_no != -1) - { - return set_directoryentry_range(blockDirectory, - columnGroupNo, - entry_no, - directoryEntry); - - } - - /* - * The given rowNum may point to a tuple that does not exist - * in the AO table any more, either because of cancellation of - * an insert, or due to crashes during an insert. If this is - * the case, rowNum is smaller than the highest entry in - * the in-memory minipage entry. - */ - else - { - MinipageEntry *entry = - &minipageInfo->minipage->entry[minipageInfo->numMinipageEntries - 1]; - - if (rowNum < entry->firstRowNum + entry->rowCount - 1) - return false; - } - } - - for (i = 0; i < blockDirectory->totalSegfiles; i++) - { - fsInfo = blockDirectory->segmentFileInfo[i]; - - if (segmentFileNum == fsInfo->segno) - break; - } - - Assert(fsInfo != NULL); - - /* - * Search the btree index to find the minipage that contains - * the rowNum. We find the minipages for all column groups, since - * currently we will need to access all columns at the same time. - */ - heapTupleDesc = RelationGetDescr(blkdirRel); - idxTupleDesc = RelationGetDescr(blkdirIdx); - - Assert(numScanKeys == 3); - - for (tmpGroupNo = 0; tmpGroupNo < blockDirectory->numColumnGroups; tmpGroupNo++) - { - /* Setup the scan keys for the scan. */ - Assert(scanKeys != NULL); - scanKeys[0].sk_argument = Int32GetDatum(segmentFileNum); - scanKeys[1].sk_argument = Int32GetDatum(tmpGroupNo); - scanKeys[2].sk_argument = Int64GetDatum(rowNum); - - idxScanDesc = index_beginscan(blkdirRel, blkdirIdx, - blockDirectory->appendOnlyMetaDataSnapshot, - numScanKeys, scanKeys); - - tuple = index_getnext(idxScanDesc, BackwardScanDirection); - - if (tuple != NULL) - { - /* - * MPP-17061: we need to update currentSegmentFileNum - * & currentSegmentFileInfo at the same time when we - * load the minipage for the block directory entry we - * found, otherwise we would risk having inconsistency - * between currentSegmentFileNum/currentSegmentFileInfo - * and minipage contents, which would cause wrong block - * header offset being returned in following block - * directory entry look up. - */ - blockDirectory->currentSegmentFileNum = segmentFileNum; - blockDirectory->currentSegmentFileInfo = fsInfo; - MinipagePerColumnGroup *minipageInfo; - - minipageInfo = &blockDirectory->minipages[tmpGroupNo]; - extract_minipage(blockDirectory, - tuple, - heapTupleDesc, - tmpGroupNo); - } - else - { - /* MPP-17061: index look up failed, row is invisible */ - index_endscan(idxScanDesc); - return false; - } - - index_endscan(idxScanDesc); - } - - { - MinipagePerColumnGroup *minipageInfo; - minipageInfo = &blockDirectory->minipages[columnGroupNo]; - - /* - * Perform a binary search over the minipage to find - * the entry about the AO block. - */ - entry_no = find_minipage_entry(minipageInfo->minipage, - minipageInfo->numMinipageEntries, - rowNum); - - /* If there are no entries, return false. */ - if (entry_no == -1 && minipageInfo->numMinipageEntries == 0) - return false; - - if (entry_no == -1) - { - /* - * Since the last few blocks may not be logged in the block - * directory, we always use the last entry. - */ - entry_no = minipageInfo->numMinipageEntries - 1; - } - return set_directoryentry_range(blockDirectory, - columnGroupNo, - entry_no, - directoryEntry); - } - - return false; -} - -/* - * AppendOnlyBlockDirectory_InsertEntry - * - * Insert an entry to the block directory. This entry is appended to the - * in-memory minipage. If the minipage is full, it is written to the block - * directory relation on disk. After that, the new entry is added to the - * new in-memory minipage. - * - * To reduce the size of a block directory, this function ignores new entries - * when the range between the offset value of the latest existing entry and - * the offset of the new entry is smaller than gp_blockdirectory_entry_min_range - * (if it is set). Otherwise, the latest existing entry is updated with new - * rowCount value, and the given new entry is appended to the in-memory minipage. - * - * If the block directory for the appendonly relation does not exist, - * this function simply returns. - * - * If rowCount is 0, simple return false. - */ -bool -AppendOnlyBlockDirectory_InsertEntry( - AppendOnlyBlockDirectory *blockDirectory, - int columnGroupNo, - int64 firstRowNum, - int64 fileOffset, - int64 rowCount) -{ - MinipageEntry *entry = NULL; - MinipagePerColumnGroup *minipageInfo = - &blockDirectory->minipages[columnGroupNo]; - int lastEntryNo; - - if (rowCount == 0) - return false; - - if (blockDirectory->blkdirRel == NULL || - blockDirectory->blkdirIdx == NULL) - return false; - - Assert(minipageInfo->numMinipageEntries <= (uint32)NUM_MINIPAGE_ENTRIES); - - lastEntryNo = minipageInfo->numMinipageEntries - 1; - if (lastEntryNo >= 0) - { - entry = &(minipageInfo->minipage->entry[lastEntryNo]); - - Assert(entry->firstRowNum < firstRowNum); - Assert(entry->fileOffset < fileOffset); - - if (gp_blockdirectory_entry_min_range > 0 && - fileOffset - entry->fileOffset < gp_blockdirectory_entry_min_range) - return true; - - /* Update the rowCount in the latest entry */ - Assert(entry->rowCount <= firstRowNum - entry->firstRowNum); - - if (Debug_appendonly_print_blockdirectory) - ereport(LOG, - (errmsg("Append-only block directory update entry: " - "(firstRowNum, columnGroupNo, fileOffset, rowCount) = (" INT64_FORMAT - ", %d, " INT64_FORMAT ", " INT64_FORMAT ") at index %d to " - "(firstRowNum, columnGroupNo, fileOffset, rowCount) = (" INT64_FORMAT - ", %d, " INT64_FORMAT ", " INT64_FORMAT ")", - entry->firstRowNum, columnGroupNo, entry->fileOffset, entry->rowCount, - minipageInfo->numMinipageEntries - 1, - entry->firstRowNum, columnGroupNo, entry->fileOffset, - firstRowNum - entry->firstRowNum))); - - entry->rowCount = firstRowNum - entry->firstRowNum; - } - - if (minipageInfo->numMinipageEntries >= (uint32)gp_blockdirectory_minipage_size) - { - write_minipage(blockDirectory, columnGroupNo); - - /* Set tupleTid to invalid */ - ItemPointerSetInvalid(&minipageInfo->tupleTid); - - /* - * Clear out the entries. - */ - MemSet(minipageInfo->minipage->entry, 0, - minipageInfo->numMinipageEntries * sizeof(MinipageEntry)); - minipageInfo->numMinipageEntries = 0; - } - - Assert(minipageInfo->numMinipageEntries < (uint32)gp_blockdirectory_minipage_size); - - entry = &(minipageInfo->minipage->entry[minipageInfo->numMinipageEntries]); - entry->firstRowNum = firstRowNum; - entry->fileOffset = fileOffset; - entry->rowCount = rowCount; - - minipageInfo->numMinipageEntries++; - - if(Debug_appendonly_print_blockdirectory) - { - ereport(LOG, - (errmsg("Append-only block directory insert entry: " - "(firstRowNum, columnGroupNo, fileOffset, rowCount) = (" INT64_FORMAT - ", %d, " INT64_FORMAT ", " INT64_FORMAT ") at index %d", - entry->firstRowNum, columnGroupNo, entry->fileOffset, entry->rowCount, - minipageInfo->numMinipageEntries - 1))); - } - - return true; -} - -/* - * init_scankeys - * - * Initialize the scan keys. - */ -static void -init_scankeys(TupleDesc tupleDesc, - int nkeys, ScanKey scanKeys, - StrategyNumber *strategyNumbers) -{ - int keyNo; - - Assert(nkeys <= tupleDesc->natts); - - for (keyNo = 0; keyNo < nkeys; keyNo ++) - { - ScanKey scanKey = (ScanKey)(((char *)scanKeys) + - keyNo * sizeof(ScanKeyData)); - RegProcedure opfuncid; - StrategyNumber strategyNumber = strategyNumbers[keyNo]; - - Assert(strategyNumber <= BTMaxStrategyNumber && - strategyNumber != InvalidStrategy); - - if (strategyNumber == BTEqualStrategyNumber) - { - opfuncid = equality_oper_funcid(tupleDesc->attrs[keyNo]->atttypid); - ScanKeyEntryInitialize(scanKey, - 0, /* sk_flag */ - keyNo + 1, /* attribute number to scan */ - BTEqualStrategyNumber, /* strategy */ - InvalidOid, /* strategy subtype */ - opfuncid, /* reg proc to use */ - 0 /* constant */ - ); - } - else - { - Oid gtOid, leOid; - - gtOid = reverse_ordering_oper_opid(tupleDesc->attrs[keyNo]->atttypid); - leOid = get_negator(gtOid); - opfuncid = get_opcode(leOid); - - ScanKeyEntryInitialize(scanKey, - 0, /* sk_flag */ - keyNo + 1, /* attribute number to scan */ - strategyNumber, /* strategy */ - InvalidOid, /* strategy subtype */ - opfuncid, /* reg proc to use */ - 0 /* constant */ - ); - } - } -} - -/* - * copy_out_minipage - * - * Copy out the minipage content from a deformed tuple. - */ -static inline void -copy_out_minipage(MinipagePerColumnGroup *minipageInfo, - Datum minipage_value, - bool minipage_isnull) -{ - struct varlena *value; - struct varlena *detoast_value; - - Assert(!minipage_isnull); - - value = (struct varlena *) - DatumGetPointer(minipage_value); - detoast_value = pg_detoast_datum(value); - Assert( VARSIZE(detoast_value) <= minipage_size(NUM_MINIPAGE_ENTRIES)); - - memcpy(minipageInfo->minipage, detoast_value, VARSIZE(detoast_value)); - if (detoast_value != value) - pfree(detoast_value); - - Assert(minipageInfo->minipage->nEntry <= NUM_MINIPAGE_ENTRIES); - - minipageInfo->numMinipageEntries = minipageInfo->minipage->nEntry; -} - - -/* - * extract_minipage - * - * Extract the minipage info from the given tuple. The tupleTid - * is also set here. - */ -static void -extract_minipage(AppendOnlyBlockDirectory *blockDirectory, - HeapTuple tuple, - TupleDesc tupleDesc, - int columnGroupNo) -{ - Datum *values = blockDirectory->values; - bool *nulls = blockDirectory->nulls; - MinipagePerColumnGroup *minipageInfo = - &blockDirectory->minipages[columnGroupNo]; - FileSegInfo *fsInfo = blockDirectory->currentSegmentFileInfo; - int64 eof; - int start, end, mid=0; - bool found = false; - - heap_deform_tuple(tuple, tupleDesc, values, nulls); - - Assert(blockDirectory->currentSegmentFileNum == - DatumGetInt32(values[Anum_pg_aoblkdir_segno - 1])); - - /* - * Copy out the minipage - */ - copy_out_minipage(minipageInfo, - values[Anum_pg_aoblkdir_minipage - 1], - nulls[Anum_pg_aoblkdir_minipage - 1]); - - ItemPointerCopy(&tuple->t_self, &minipageInfo->tupleTid); - - /* - * When crashes during inserts, or cancellation during inserts, - * there are out-of-date minipage entries in the block directory. - * We reset those entries here. - */ - Assert(fsInfo != NULL); - eof = fsInfo->eof; - - start = 0; - end = minipageInfo->numMinipageEntries - 1; - while (start <= end) - { - mid = (end - start + 1) / 2 + start; - if (minipageInfo->minipage->entry[mid].fileOffset > eof) - end = mid - 1; - else if (minipageInfo->minipage->entry[mid].fileOffset < eof) - start = mid + 1; - else - { - found = true; - break; - } - } - - minipageInfo->numMinipageEntries = 0; - if (found) - minipageInfo->numMinipageEntries = mid; - else if (start > 0) - { - minipageInfo->numMinipageEntries = start; - Assert(minipageInfo->minipage->entry[start - 1].fileOffset < eof); - } -} - -/* - * load_last_minipage - * - * Search through the block directory btree to find the last row that - * contains the last minipage. - */ -static void -load_last_minipage(AppendOnlyBlockDirectory *blockDirectory, - int64 lastSequence, - int columnGroupNo) -{ - Relation blkdirRel = blockDirectory->blkdirRel; - Relation blkdirIdx = blockDirectory->blkdirIdx; - TupleDesc idxTupleDesc; - TupleDesc heapTupleDesc; - IndexScanDesc idxScanDesc; - HeapTuple tuple = NULL; - MemoryContext oldcxt; - int numScanKeys = blockDirectory->numScanKeys; - ScanKey scanKeys = blockDirectory->scanKeys; - -#ifdef USE_ASSERT_CHECKING - StrategyNumber *strategyNumbers = blockDirectory->strategyNumbers; -#endif /* USE_ASSERT_CHECKING */ - - Assert(blockDirectory->aoRel != NULL); - Assert(blockDirectory->blkdirRel != NULL); - Assert(blockDirectory->blkdirIdx != NULL); - - oldcxt = MemoryContextSwitchTo(blockDirectory->memoryContext); - - heapTupleDesc = RelationGetDescr(blkdirRel); - idxTupleDesc = RelationGetDescr(blkdirIdx); - - Assert(numScanKeys == 3); - Assert(blockDirectory->currentSegmentFileInfo != NULL); - - /* Setup the scan keys for the scan. */ - Assert(scanKeys != NULL); - Assert(strategyNumbers != NULL); - if (lastSequence == 0) - lastSequence = 1; - - scanKeys[0].sk_argument = - Int32GetDatum(blockDirectory->currentSegmentFileNum); - scanKeys[1].sk_argument = Int32GetDatum(columnGroupNo); - scanKeys[2].sk_argument = Int64GetDatum(lastSequence); - - /* - * Search the btree to find the entry in the block directory - * that contains the last minipage. - */ - idxScanDesc = index_beginscan(blkdirRel, blkdirIdx, - blockDirectory->appendOnlyMetaDataSnapshot, - numScanKeys, scanKeys); - - tuple = index_getnext(idxScanDesc, BackwardScanDirection); - if (tuple != NULL) - { - extract_minipage(blockDirectory, - tuple, - heapTupleDesc, - columnGroupNo); - } - - index_endscan(idxScanDesc); - - MemoryContextSwitchTo(oldcxt); - - if (Debug_appendonly_print_blockdirectory) - ereport(LOG, - (errmsg("Append-only block directory load last minipage: " - "(columnGroupNo, lastSequence, nEntries) = (%d, " INT64_FORMAT ", %u)", - columnGroupNo, lastSequence, - blockDirectory->minipages[columnGroupNo].numMinipageEntries))); - -} - -/* - * find_minipage_entry - * - * Find the minipage entry that covers the given rowNum. - * If such an entry does not exists, -1 is returned. Otherwise - * the index to such an entry in the minipage array is returned. - */ -static int -find_minipage_entry(Minipage *minipage, - uint32 numEntries, - int64 rowNum) -{ - int start_no, end_no; - int entry_no; - MinipageEntry *entry; - - start_no = 0; - end_no = numEntries - 1; - while (start_no <= end_no) - { - entry_no = start_no + (end_no - start_no + 1) / 2; - Assert(entry_no >= start_no && entry_no <= end_no); - - entry = &(minipage->entry[entry_no]); - - Assert(entry->firstRowNum > 0); - Assert(entry->rowCount > 0); - - if (entry->firstRowNum <= rowNum && - entry->firstRowNum + entry->rowCount > rowNum) - break; - else if (entry->firstRowNum > rowNum) - { - end_no = entry_no - 1; - } - else - { - start_no = entry_no + 1; - } - } - - if (start_no <= end_no) - return entry_no; - else - return -1; -} - -/* - * write_minipage - * - * Write the in-memory minipage to the block directory relation. - */ -static void -write_minipage(AppendOnlyBlockDirectory *blockDirectory, - int columnGroupNo) -{ - HeapTuple tuple; - MemoryContext oldcxt; - Datum *values = blockDirectory->values; - bool *nulls = blockDirectory->nulls; - Relation blkdirRel = blockDirectory->blkdirRel; - TupleDesc heapTupleDesc = RelationGetDescr(blkdirRel); - MinipagePerColumnGroup *minipageInfo = - &blockDirectory->minipages[columnGroupNo]; - - Assert(minipageInfo->numMinipageEntries > 0); - - oldcxt = MemoryContextSwitchTo(blockDirectory->memoryContext); - - Assert(blkdirRel != NULL); - - values[Anum_pg_aoblkdir_segno - 1] = - Int32GetDatum(blockDirectory->currentSegmentFileNum); - nulls[Anum_pg_aoblkdir_segno - 1] = false; - - values[Anum_pg_aoblkdir_columngroupno - 1] = - Int32GetDatum(columnGroupNo); - nulls[Anum_pg_aoblkdir_columngroupno - 1] = false; - - values[Anum_pg_aoblkdir_firstrownum - 1] = - Int64GetDatum(minipageInfo->minipage->entry[0].firstRowNum); - nulls[Anum_pg_aoblkdir_firstrownum - 1] = false; - - SET_VARSIZE(minipageInfo->minipage, - minipage_size(minipageInfo->numMinipageEntries)); - minipageInfo->minipage->nEntry = minipageInfo->numMinipageEntries; - values[Anum_pg_aoblkdir_minipage - 1] = - PointerGetDatum(minipageInfo->minipage); - nulls[Anum_pg_aoblkdir_minipage - 1] = false; - - tuple = heaptuple_form_to(heapTupleDesc, - values, - nulls, - NULL, - NULL); - - /* - * Write out the minipage to the block directory relation. - * If this minipage is already in the relation, we update - * the row. Otherwise, a new row is inserted. - */ - if (ItemPointerIsValid(&minipageInfo->tupleTid)) - { - if (Debug_appendonly_print_blockdirectory) - ereport(LOG, - (errmsg("Append-only block directory update a minipage: " - "(segno, columnGroupNo, nEntries, firstRowNum) = " - "(%d, %d, %u, " INT64_FORMAT ")", - blockDirectory->currentSegmentFileNum, - columnGroupNo, minipageInfo->numMinipageEntries, - minipageInfo->minipage->entry[0].firstRowNum))); - - simple_heap_update(blkdirRel, &minipageInfo->tupleTid, tuple); - } - else - { - if (Debug_appendonly_print_blockdirectory) - ereport(LOG, - (errmsg("Append-only block directory insert a minipage: " - "(segno, columnGroupNo, nEntries, firstRowNum) = " - "(%d, %d, %u, " INT64_FORMAT ")", - blockDirectory->currentSegmentFileNum, - columnGroupNo, minipageInfo->numMinipageEntries, - minipageInfo->minipage->entry[0].firstRowNum))); - - simple_heap_insert(blkdirRel, tuple); - } - - CatalogUpdateIndexes(blkdirRel, tuple); - - heap_freetuple(tuple); - - MemoryContextSwitchTo(oldcxt); -} - - - -void -AppendOnlyBlockDirectory_End_forInsert( - AppendOnlyBlockDirectory *blockDirectory) -{ - int groupNo; - - if (blockDirectory->blkdirRel == NULL || - blockDirectory->blkdirIdx == NULL) - return; - - for (groupNo = 0; groupNo < blockDirectory->numColumnGroups; groupNo++) - { - MinipagePerColumnGroup *minipageInfo = - &blockDirectory->minipages[groupNo]; - - if (minipageInfo->numMinipageEntries > 0) - { - write_minipage(blockDirectory, groupNo); - if (Debug_appendonly_print_blockdirectory) - ereport(LOG, - (errmsg("Append-only block directory end of insert write minipage: " - "(columnGroupNo, nEntries) = (%d, %u)", - groupNo, minipageInfo->numMinipageEntries))); - } - - pfree(minipageInfo->minipage); - } - - if (Debug_appendonly_print_blockdirectory) - ereport(LOG, - (errmsg("Append-only block directory end for insert: " - "(segno, numColumnGroups)=" - "(%d, %d)", - blockDirectory->currentSegmentFileNum, - blockDirectory->numColumnGroups))); - - pfree(blockDirectory->values); - pfree(blockDirectory->nulls); - pfree(blockDirectory->minipages); - pfree(blockDirectory->scanKeys); - pfree(blockDirectory->strategyNumbers); - - index_close(blockDirectory->blkdirIdx, RowExclusiveLock); - heap_close(blockDirectory->blkdirRel, RowExclusiveLock); - - MemoryContextDelete(blockDirectory->memoryContext); -} - -void -AppendOnlyBlockDirectory_End_forSearch( - AppendOnlyBlockDirectory *blockDirectory) -{ - int groupNo; - - if (blockDirectory->blkdirRel == NULL || - blockDirectory->blkdirIdx == NULL) - return; - - for (groupNo = 0; groupNo < blockDirectory->numColumnGroups; groupNo++) - { - pfree(blockDirectory->minipages[groupNo].minipage); - } - - if (Debug_appendonly_print_blockdirectory) - ereport(LOG, - (errmsg("Append-only block directory end for search: " - "(totalSegfiles, numColumnGroups )=" - "(%d, %d)", - blockDirectory->totalSegfiles, - blockDirectory->numColumnGroups))); - - pfree(blockDirectory->values); - pfree(blockDirectory->nulls); - pfree(blockDirectory->minipages); - pfree(blockDirectory->scanKeys); - pfree(blockDirectory->strategyNumbers); - - index_close(blockDirectory->blkdirIdx, AccessShareLock); - heap_close(blockDirectory->blkdirRel, AccessShareLock); - - MemoryContextDelete(blockDirectory->memoryContext); -} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae38cfbd/src/backend/access/external/hd_work_mgr.c ---------------------------------------------------------------------- diff --git a/src/backend/access/external/hd_work_mgr.c b/src/backend/access/external/hd_work_mgr.c index b5bf6b5..c7c3e20 100644 --- a/src/backend/access/external/hd_work_mgr.c +++ b/src/backend/access/external/hd_work_mgr.c @@ -242,8 +242,6 @@ static void assign_pxf_port_to_fragments(int remote_rest_port, List *fragments) PxfFragmentStatsElem *get_pxf_fragments_statistics(char *uri, Relation rel) { ClientContext client_context; /* holds the communication info */ - char *analyzer = NULL; - char *profile = NULL; PxfInputData inputData = {0}; PxfFragmentStatsElem *result = NULL; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae38cfbd/src/backend/access/heap/heapam.c ---------------------------------------------------------------------- diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index b4d4e5c..78427e9 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -50,7 +50,6 @@ #include "catalog/catalog.h" #include "catalog/catquery.h" #include "catalog/gp_policy.h" -#include "catalog/gp_fastsequence.h" #include "catalog/namespace.h" #include "miscadmin.h" #include "pgstat.h" @@ -4888,42 +4887,6 @@ heap_xlog_insert(XLogRecPtr lsn, XLogRecord *record) HeapTupleHeaderSetCmin(htup, FirstCommandId); htup->t_ctid = xlrec->target.tid; - /* MPP-16881: if GPDB crashed when CTAS for AO/CO is running, we should "frozen" - * delete left-over entries in gp_fastsequence catalog table when replaying XLOG - * during Pass3 crash recovery. - */ - if (xlrec->target.node.relNode == FastSequenceRelationId) - { - Form_gp_fastsequence redo_entry = - (Form_gp_fastsequence) ((char *) htup + htup->t_hoff); - HASH_SEQ_STATUS iterateStatus; - hash_seq_init(&iterateStatus, pass2RecoveryHashShmem->hash); - - while (true) - { - Pass2RecoveryHashEntry_s *entry = - (Pass2RecoveryHashEntry_s *) - hash_seq_search(&iterateStatus); - - if (entry == NULL) - break; - - if (entry->objid == redo_entry->objid) - { - htup->t_infomask &= 0; - htup->t_infomask |= HEAP_XMIN_INVALID; - HeapTupleHeaderSetXmin(htup, 0); - HeapTupleHeaderSetXmax(htup, FrozenTransactionId); - hash_seq_term(&iterateStatus); - if (Debug_persistent_print) - elog(LOG, "frozen deleting gp_fastsequence entry" - "for aborted AO insert transaction on objid %d", - redo_entry->objid); - break; - } - } - } - offnum = PageAddItem(page, (Item) htup, newlen, offnum, LP_USED | OverwritePageMode); if (offnum == InvalidOffsetNumber) http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae38cfbd/src/backend/access/index/caql.files ---------------------------------------------------------------------- diff --git a/src/backend/access/index/caql.files b/src/backend/access/index/caql.files index 3edb42c..412aa24 100644 --- a/src/backend/access/index/caql.files +++ b/src/backend/access/index/caql.files @@ -4,7 +4,6 @@ ../../../..//src/backend/catalog/caql/caqlanalyze.c ../../../..//src/backend/catalog/caql/test/caqlanalyze_test.c ../../../..//src/backend/catalog/dependency.c -../../../..//src/backend/catalog/gp_fastsequence.c ../../../..//src/backend/catalog/heap.c ../../../..//src/backend/catalog/index.c ../../../..//src/backend/catalog/namespace.c http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae38cfbd/src/backend/access/index/caqlfilemap.json ---------------------------------------------------------------------- diff --git a/src/backend/access/index/caqlfilemap.json b/src/backend/access/index/caqlfilemap.json index 8ff325c..7e9e089 100644 --- a/src/backend/access/index/caqlfilemap.json +++ b/src/backend/access/index/caqlfilemap.json @@ -152,9 +152,6 @@ "functions.c" : [ "pg_proc" ], - "gp_fastsequence.c" : [ - "gp_fastsequence" - ], "heap.c" : [ "pg_attrdef", "pg_attribute", http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae38cfbd/src/backend/access/nbtree/nbtinsert.c ---------------------------------------------------------------------- diff --git a/src/backend/access/nbtree/nbtinsert.c b/src/backend/access/nbtree/nbtinsert.c index 24f7bd9..92ea375 100644 --- a/src/backend/access/nbtree/nbtinsert.c +++ b/src/backend/access/nbtree/nbtinsert.c @@ -162,51 +162,6 @@ top: } /* - * _bt_ao_check_unique -- Check for violation of unique index constraint - * for AO tables. - * - * Returns InvalidTransactionId if there is no conflict, else an xact ID - * we must wait for to see if it commits a conflicting tuple. If an actual - * conflict is detected, no return --- just ereport(). - */ -static TransactionId -_bt_ao_check_unique(Relation rel, Relation aoRel, ItemPointer tid) -{ - TransactionId xwait = InvalidTransactionId; - - Assert(RelationIsAoRows(aoRel)); - - if (RelationIsAoRows(aoRel)) - { - AppendOnlyFetchDesc aoFetchDesc = - appendonly_fetch_init(aoRel, SnapshotDirty); - - if (appendonly_fetch(aoFetchDesc, (AOTupleId*)tid, NULL)) - { - xwait = - (TransactionIdIsValid(SnapshotDirty->xmin)) ? - SnapshotDirty->xmin : SnapshotDirty->xmax; - - /* - * If this tuple is not being updated by other transaction, - * then we have a definite conflict. Ereport here. - */ - if (!TransactionIdIsValid(xwait)) - ereport(ERROR, - (errcode(ERRCODE_UNIQUE_VIOLATION), - errmsg("duplicate key violates unique constraint \"%s\"", - RelationGetRelationName(rel)), - errOmitLocation(true))); - } - - appendonly_fetch_finish(aoFetchDesc); - pfree(aoFetchDesc); - } - - return xwait; -} - -/* * _bt_check_unique() -- Check for violation of unique index constraint * * Returns InvalidTransactionId if there is no conflict, else an xact ID @@ -288,16 +243,6 @@ _bt_check_unique(Relation rel, IndexTuple itup, Relation heapRel, * If the parent relation is an AO/CO table, we have to find out * if this tuple is actually in the table. */ - if (RelationIsAoRows(heapRel)) - { - TransactionId xwait = - _bt_ao_check_unique(rel, heapRel, &curitup->t_tid); - - if (TransactionIdIsValid(xwait)) - return xwait; - } - - else { htup.t_self = curitup->t_tid; if (heap_fetch(heapRel, SnapshotDirty, &htup, &hbuffer, http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae38cfbd/src/backend/access/parquet/parquetsegfiles.c ---------------------------------------------------------------------- diff --git a/src/backend/access/parquet/parquetsegfiles.c b/src/backend/access/parquet/parquetsegfiles.c index cf8d140..3404a9e 100644 --- a/src/backend/access/parquet/parquetsegfiles.c +++ b/src/backend/access/parquet/parquetsegfiles.c @@ -35,7 +35,6 @@ #include "catalog/dependency.h" #include "catalog/indexing.h" #include "catalog/namespace.h" -#include "catalog/gp_fastsequence.h" #include "catalog/pg_appendonly.h" #include "cdb/cdbvars.h" #include "executor/spi.h" @@ -191,13 +190,9 @@ void InsertInitialParquetSegnoEntry(AppendOnlyEntry *aoEntry, int segno) { int natts = 0; bool *nulls; Datum *values; - ItemPointerData tid; Assert(aoEntry != NULL); - InsertFastSequenceEntry(aoEntry->segrelid, (int64) segno, 0, - &tid); - if (segno == 0) { return; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae38cfbd/src/backend/bootstrap/bootparse.y ---------------------------------------------------------------------- diff --git a/src/backend/bootstrap/bootparse.y b/src/backend/bootstrap/bootparse.y index 4e9ff00..74f8471 100755 --- a/src/backend/bootstrap/bootparse.y +++ b/src/backend/bootstrap/bootparse.y @@ -47,7 +47,6 @@ #include "catalog/gp_configuration.h" #include "catalog/gp_persistent.h" #include "catalog/gp_global_sequence.h" -#include "catalog/gp_fastsequence.h" #include "catalog/gp_san_config.h" #include "catalog/gp_segment_config.h" #include "catalog/gp_verification_history.h" @@ -317,10 +316,6 @@ Boot_CreateStmt: case GpFaultStrategyRelationId: typid = GP_FAULT_STRATEGY_RELTYPE_OID; break; -/* relation id: 5043 - gp_fastsequence 20101104 */ - case FastSequenceRelationId: - typid = GP_FASTSEQUENCE_RELTYPE_OID; - break; /* relation id: 5096 - gp_global_sequence 20101104 */ case GpGlobalSequenceRelationId: typid = GP_GLOBAL_SEQUENCE_RELTYPE_OID; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae38cfbd/src/backend/catalog/Makefile ---------------------------------------------------------------------- diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index 54d26cb..be3e5da 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -19,7 +19,7 @@ OBJS = catalog.o dependency.o heap.o index.o indexing.o namespace.o aclchk.o \ pg_aggregate.o pg_appendonly.o pg_constraint.o pg_conversion.o pg_depend.o \ pg_exttable.o pg_extprotocol.o pg_filesystem.o pg_largeobject.o pg_namespace.o \ pg_operator.o pg_proc.o pg_proc_callback.o pg_shdepend.o \ - pg_type.o toasting.o aoseg.o aoblkdir.o gp_fastsequence.o \ + pg_type.o toasting.o aoseg.o \ pg_attribute_encoding.o pg_compression.o $(QUICKLZ_COMPRESSION) SUBDIRS = caql core hcatalog @@ -113,7 +113,7 @@ POSTGRES_BKI_SRCS := $(addprefix $(top_srcdir)/src/include/catalog/,\ gp_verification_history.h \ pg_window.h \ pg_exttable.h pg_appendonly.h pg_appendonly_alter_column.h gp_master_mirroring.h aoseg.h \ - gp_fastsequence.h pg_extprotocol.h \ + pg_extprotocol.h \ pg_partition.h pg_partition_rule.h pg_filespace.h pg_filespace_entry.h \ gp_global_sequence.h gp_persistent.h \ $(TIDYCAT_BKI_SRCS) \ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae38cfbd/src/backend/catalog/aoblkdir.c ---------------------------------------------------------------------- diff --git a/src/backend/catalog/aoblkdir.c b/src/backend/catalog/aoblkdir.c deleted file mode 100644 index 23863e0..0000000 --- a/src/backend/catalog/aoblkdir.c +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/*------------------------------------------------------------------------- - * - * aoblkdir.c - * This file contains routines to support creation of append-only block - * directory tables. This file is identical in functionality to aoseg.c - * that exists in the same directory. - * - *------------------------------------------------------------------------- - */ -#include "postgres.h" - -#include "access/heapam.h" -#include "access/xact.h" -#include "catalog/dependency.h" -#include "catalog/heap.h" -#include "catalog/index.h" -#include "catalog/indexing.h" -#include "catalog/pg_namespace.h" -#include "catalog/pg_opclass.h" -#include "catalog/pg_type.h" -#include "commands/tablespace.h" -#include "miscadmin.h" -#include "nodes/makefuncs.h" -#include "utils/builtins.h" -#include "utils/syscache.h" -#include "catalog/aoblkdir.h" - - -static bool -create_aoblkdir_table(Relation rel, Oid aoblkdirOid, - Oid aoblkdirIndexOid, Oid *comptypeOid); - -/* - * AlterTableCreateAoBlkdirTable - * If the table needs an AO block directory table, and doesn't already - * have one, then create an aoblkdir table. - * - * We expect the caller to have verified that the relation is AO table and have - * already done any necessary permission checks. Callers expect this function - * to end with CommandCounterIncrement if it makes any changes. - */ -void -AlterTableCreateAoBlkdirTable(Oid relOid) -{ - Relation rel; - - /* - * We've well and truly locked the table if we need to, so don't now. - * This is useful for avoiding lock table overflows with partitioning. - */ - rel = heap_open(relOid, NoLock); - - /* create_aoblkdir_table does all the work */ - (void) create_aoblkdir_table(rel, InvalidOid, InvalidOid, NULL); - - heap_close(rel, NoLock); -} - -void -AlterTableCreateAoBlkdirTableWithOid(Oid relOid, Oid newOid, Oid newIndexOid, - Oid * comptypeOid, bool is_part_child) -{ - Relation rel; - - /* - * Grab an exclusive lock on the target table, which we will NOT release - * until end of transaction. (This is probably redundant in all present - * uses...) - */ - if (is_part_child) - rel = heap_open(relOid, NoLock); - else - rel = heap_open(relOid, AccessExclusiveLock); - - /* create_aoblkdir_table does all the work */ - (void) create_aoblkdir_table(rel, newOid, newIndexOid, comptypeOid); - - heap_close(rel, NoLock); -} - -/* - * create_aoblkdir_table - * - * rel is already opened and exclusive-locked. - * comptypeOid is InvalidOid. - */ -static bool -create_aoblkdir_table(Relation rel, Oid aoblkdirOid, - Oid aoblkdirIndexOid, Oid *comptypeOid) -{ - Oid relOid = RelationGetRelid(rel); - Oid aoblkdir_relid; - Oid aoblkdir_idxid; - bool shared_relation = rel->rd_rel->relisshared; - char aoblkdir_relname[NAMEDATALEN]; - char aoblkdir_idxname[NAMEDATALEN]; - TupleDesc tupdesc; - IndexInfo *indexInfo; - Oid classObjectId[3]; - ObjectAddress baseobject; - ObjectAddress aoblkdirobject; - Oid tablespaceOid = ChooseTablespaceForLimitedObject(rel->rd_rel->reltablespace); - - if (!RelationIsAoRows(rel)) - return false; - - /* - * We cannot allow creating a block directory for a shared relation - * after initdb (because there's no way to let other databases know - * this block directory. - */ - if (shared_relation && !IsBootstrapProcessingMode()) - ereport(ERROR, - (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("shared tables cannot have block directory after initdb"))); - - GetAppendOnlyEntryAuxOids(relOid, SnapshotNow, NULL,NULL, &aoblkdir_relid, &aoblkdir_idxid); - - /* - * Does it have a block directory? - */ - if (aoblkdir_relid != InvalidOid) - { - return false; - } - - snprintf(aoblkdir_relname, sizeof(aoblkdir_relname), - "pg_aoblkdir_%u", relOid); - snprintf(aoblkdir_idxname, sizeof(aoblkdir_idxname), - "pg_aoblkdir_%u_index", relOid); - - /* Create a tuple descriptor */ - tupdesc = CreateTemplateTupleDesc(4, false); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, - "segno", - INT4OID, - -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 2, - "columngroup_no", - INT4OID, - -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, - "first_row_no", - INT8OID, - -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, - "minipage", - VARBITOID, - -1, 0); - /* - * We don't want any toast columns here. - */ - tupdesc->attrs[0]->attstorage = 'p'; - tupdesc->attrs[1]->attstorage = 'p'; - tupdesc->attrs[2]->attstorage = 'p'; - tupdesc->attrs[2]->attstorage = 'p'; - - /* - * We place aoblkdir relation in the pg_aoseg namespace - * even if its master relation is a temp table. There cannot be - * any naming collision, and the aoblkdir relation will be - * destroyed when its master is, so there is no need to handle - * the aoblkdir relation as temp. - */ - aoblkdir_relid = heap_create_with_catalog(aoblkdir_relname, - PG_AOSEGMENT_NAMESPACE, - tablespaceOid, - aoblkdirOid, - rel->rd_rel->relowner, - tupdesc, - /* relam */ InvalidOid, - RELKIND_AOBLOCKDIR, - RELSTORAGE_HEAP, - shared_relation, - true, - /* bufferPoolBulkLoad */ false, - 0, - ONCOMMIT_NOOP, - NULL, /* GP Policy */ - (Datum) 0, - true, - comptypeOid, - /* persistentTid */ NULL, - /* persistentSerialNum */ NULL); - - /* Make this table visible, else index creation will fail */ - CommandCounterIncrement(); - - /* - * Create index on segno, first_row_no. - */ - indexInfo = makeNode(IndexInfo); - indexInfo->ii_NumIndexAttrs = 3; - indexInfo->ii_KeyAttrNumbers[0] = 1; - indexInfo->ii_KeyAttrNumbers[1] = 2; - indexInfo->ii_KeyAttrNumbers[2] = 3; - indexInfo->ii_Expressions = NIL; - indexInfo->ii_ExpressionsState = NIL; - indexInfo->ii_Predicate = NIL; - indexInfo->ii_PredicateState = NIL; - indexInfo->ii_Unique = false; - indexInfo->ii_Concurrent = false; - - classObjectId[0] = INT4_BTREE_OPS_OID; - classObjectId[1] = INT4_BTREE_OPS_OID; - classObjectId[2] = INT8_BTREE_OPS_OID; - - aoblkdir_idxid = index_create(aoblkdirOid, aoblkdir_idxname, aoblkdirIndexOid, - indexInfo, - BTREE_AM_OID, - tablespaceOid, - classObjectId, (Datum) 0, - true, false, (Oid *) NULL, true, false, false, NULL); - - /* Unlock target table -- no one can see it */ - UnlockRelationOid(aoblkdirOid, ShareLock); - /* Unlock the index -- no one can see it anyway */ - UnlockRelationOid(aoblkdirIndexOid, AccessExclusiveLock); - - /* - * Store the aoblkdir table's OID in the parent relation's pg_appendonly row. - */ - UpdateAppendOnlyEntryAuxOids(relOid, InvalidOid, InvalidOid, - aoblkdir_relid, aoblkdir_idxid); - - /* - * Register dependency from the aoseg table to the master, so that the - * aoseg table will be deleted if the master is. - */ - baseobject.classId = RelationRelationId; - baseobject.objectId = relOid; - baseobject.objectSubId = 0; - aoblkdirobject.classId = RelationRelationId; - aoblkdirobject.objectId = aoblkdirOid; - aoblkdirobject.objectSubId = 0; - - recordDependencyOn(&aoblkdirobject, &baseobject, DEPENDENCY_INTERNAL); - - /* - * Make changes visible - */ - CommandCounterIncrement(); - - return true; -} - http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae38cfbd/src/backend/catalog/caql/Makefile ---------------------------------------------------------------------- diff --git a/src/backend/catalog/caql/Makefile b/src/backend/catalog/caql/Makefile index 77bcb5c..67c1876 100644 --- a/src/backend/catalog/caql/Makefile +++ b/src/backend/catalog/caql/Makefile @@ -37,7 +37,6 @@ CAQL_CQL_SRCS := $(addprefix $(top_srcdir)/src/backend/,\ access/heap/heapam.c \ catalog/aclchk.c \ catalog/dependency.c \ - catalog/gp_fastsequence.c \ catalog/heap.c \ catalog/index.c \ catalog/namespace.c \ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae38cfbd/src/backend/catalog/core/catcoregen.py ---------------------------------------------------------------------- diff --git a/src/backend/catalog/core/catcoregen.py b/src/backend/catalog/core/catcoregen.py index 1c725fe..16c0f31 100644 --- a/src/backend/catalog/core/catcoregen.py +++ b/src/backend/catalog/core/catcoregen.py @@ -420,7 +420,6 @@ CatCoreTableTemplate = """ #include "catalog/catcore.h" #include "catalog/catalog.h" #include "catalog/gp_configuration.h" -#include "catalog/gp_fastsequence.h" #include "catalog/gp_id.h" #include "catalog/gp_master_mirroring.h" #include "catalog/gp_policy.h"
