>From Ritik Raj <[email protected]>: Ritik Raj has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20151 )
Change subject: [NO ISSUE][STO] Fixed multiple issue ...................................................................... [NO ISSUE][STO] Fixed multiple issue - user model changes: no - storage format changes: no - interface changes: no Details: 1. Resetting to current pageZero, when pinning pageZero extra segments. 2. Issue while reading the pageZero segments because of incorrect column count. 3. Fixed merge issue, where we will store less effective columns, but the number of columns stored in the pageZero is the global number of columns Ext-ref: MB-67802 Change-Id: I1a5725e36301537a0926781d1d875ff62d523545 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20151 Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Peeyush Gupta <[email protected]> --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/ColumnRanges.java M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/MultiPageZeroByteBuffersReader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/MergeColumnTupleReference.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudMegaPageReadContext.java M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleWriter.java 7 files changed, 55 insertions(+), 67 deletions(-) Approvals: Peeyush Gupta: Looks good to me, approved Jenkins: Verified; Verified Anon. E. Moose #1000171: diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/MultiPageZeroByteBuffersReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/MultiPageZeroByteBuffersReader.java index 1ab3052..a5a7477 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/MultiPageZeroByteBuffersReader.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/MultiPageZeroByteBuffersReader.java @@ -86,8 +86,7 @@ } public int readOffset(long[] offsetColumnIndexPairs, int maxColumnsInZerothSegment, int numberOfColumnsInAPage, - int currentColumnIndex) { - int numberOfColumns = offsetColumnIndexPairs.length - 1; + int currentColumnIndex, int numberOfColumns) { for (Int2IntMap.Entry pair : segmentDir.int2IntEntrySet()) { int segmentIndex = pair.getIntKey(); int bufferIndex = pair.getIntValue(); diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleWriter.java index cce694d..9feb789 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleWriter.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnTupleWriter.java @@ -60,7 +60,6 @@ private final int maxNumberOfTuples; private int primaryKeysEstimatedSize; private int numberOfAntiMatter; - private int numberOfTuples; public MergeColumnTupleWriter(MergeColumnWriteMetadata columnMetadata, int pageSize, int maxNumberOfTuples, double tolerance, int maxLeafNodeSize, IColumnWriteContext writeContext) { @@ -68,6 +67,8 @@ this.pageZeroWriterFlavorSelector = new PageZeroWriterFlavorSelector(); this.maxLeafNodeSize = maxLeafNodeSize; this.presentColumnsIndexes = new BitSet(); + int numberOfColumns = columnMetadata.getNumberOfColumns(); + presentColumnsIndexes.set(0, numberOfColumns); List<IColumnTupleIterator> componentsTuplesList = columnMetadata.getComponentsTuples(); this.componentsTuples = new MergeColumnTupleReference[componentsTuplesList.size()]; int totalLength = 0; @@ -76,7 +77,6 @@ MergeColumnTupleReference mergeTuple = (MergeColumnTupleReference) componentsTuplesList.get(i); this.componentsTuples[i] = mergeTuple; mergeTuple.registerEndOfPageCallBack(this::writeAllColumns); - mergeTuple.setColumnIndexes(presentColumnsIndexes); totalNumberOfTuples += mergeTuple.getTupleCount(); totalLength += mergeTuple.getMergingLength(); } @@ -125,13 +125,6 @@ @Override public void writeTuple(ITupleReference tuple) throws HyracksDataException { MergeColumnTupleReference columnTuple = (MergeColumnTupleReference) tuple; - if (numberOfTuples == 0) { - // fill with the columnIndexes - for (MergeColumnTupleReference componentsTuple : componentsTuples) { - componentsTuple.fillColumnIndexes(); - } - } - numberOfTuples++; int componentIndex = columnTuple.getComponentIndex(); int skipCount = columnTuple.getAndResetSkipCount(); if (skipCount > 0) { @@ -189,17 +182,17 @@ } private int getSpaceOccupiedBySparseWriter(int maxColumnsInPageZerothSegment, int bufferCapacity) { - int presentColumns = presentColumnsIndexes.cardinality(); + int numberOfColumns = columnMetadata.getNumberOfColumns(); int maximumNumberOfColumnsInASegment = SparseColumnMultiPageZeroWriter.getMaximumNumberOfColumnsInAPage(bufferCapacity); - int numberOfExtraPagesRequired = presentColumns <= maxColumnsInPageZerothSegment ? 0 + int numberOfExtraPagesRequired = numberOfColumns <= maxColumnsInPageZerothSegment ? 0 : (int) Math.ceil( - (double) (presentColumns - maxColumnsInPageZerothSegment) / maximumNumberOfColumnsInASegment); + (double) (numberOfColumns - maxColumnsInPageZerothSegment) / maximumNumberOfColumnsInASegment); int headerSpace = SparseColumnMultiPageZeroWriter.getHeaderSpace(numberOfExtraPagesRequired); - presentColumns = Math.min(presentColumns, maxColumnsInPageZerothSegment); + numberOfColumns = Math.min(numberOfColumns, maxColumnsInPageZerothSegment); // space occupied by the sparse writer - return headerSpace + presentColumns + return headerSpace + numberOfColumns * (SparseColumnPageZeroWriter.COLUMN_OFFSET_SIZE + DefaultColumnPageZeroWriter.FILTER_SIZE); } @@ -216,14 +209,8 @@ writeNonKeyColumns(); writtenComponents.reset(); } - - // Iterate over the BitSet (presentColumnsIndexes) to get the indexes of the set bits - for (int columnIndex = presentColumnsIndexes.nextSetBit(0); columnIndex >= 0; columnIndex = - presentColumnsIndexes.nextSetBit(columnIndex + 1)) { - if (columnIndex < numberOfPrimaryKeys) { - continue; // Skip primary key columns - } - orderedColumns.add(columnMetadata.getWriter(columnIndex)); + for (int i = numberOfPrimaryKeys; i < numberOfColumns; i++) { + orderedColumns.add(columnMetadata.getWriter(i)); } // Reset pageZeroWriter based on the writer @@ -258,8 +245,6 @@ @Override public void reset() { - presentColumnsIndexes.clear(); - numberOfTuples = 0; } private void writePrimaryKeys(MergeColumnTupleReference columnTuple) throws HyracksDataException { @@ -277,22 +262,16 @@ for (int i = 0; i < writtenComponents.getNumberOfBlocks(); i++) { int componentIndex = writtenComponents.getBlockValue(i); if (componentIndex < 0) { - // Skip writing values of deleted tuples + //Skip writing values of deleted tuples componentIndex = clearAntimatterIndicator(componentIndex); skipReaders(componentIndex, writtenComponents.getBlockSize(i)); continue; } MergeColumnTupleReference componentTuple = componentsTuples[componentIndex]; int count = writtenComponents.getBlockSize(i); - - // Iterate over the set bits in presentColumnsIndexes - for (int columnIndex = presentColumnsIndexes.nextSetBit(0); columnIndex >= 0; columnIndex = - presentColumnsIndexes.nextSetBit(columnIndex + 1)) { - if (columnIndex < columnMetadata.getNumberOfPrimaryKeys()) { - continue; - } - IColumnValuesReader columnReader = componentTuple.getReader(columnIndex); - IColumnValuesWriter columnWriter = columnMetadata.getWriter(columnIndex); + for (int j = columnMetadata.getNumberOfPrimaryKeys(); j < columnMetadata.getNumberOfColumns(); j++) { + IColumnValuesReader columnReader = componentTuple.getReader(j); + IColumnValuesWriter columnWriter = columnMetadata.getWriter(j); writeColumn(i, componentIndex, columnReader, columnWriter, count); } } @@ -313,13 +292,8 @@ private void skipReaders(int componentIndex, int count) throws HyracksDataException { MergeColumnTupleReference componentTuple = componentsTuples[componentIndex]; try { - // Iterate over the set bits in presentColumnsIndexes - for (int columnIndex = presentColumnsIndexes.nextSetBit(0); columnIndex >= 0; columnIndex = - presentColumnsIndexes.nextSetBit(columnIndex + 1)) { - if (columnIndex < columnMetadata.getNumberOfPrimaryKeys()) { - continue; - } - IColumnValuesReader columnReader = componentTuple.getReader(columnIndex); + for (int j = columnMetadata.getNumberOfPrimaryKeys(); j < columnMetadata.getNumberOfColumns(); j++) { + IColumnValuesReader columnReader = componentTuple.getReader(j); columnReader.skip(count); } } catch (ColumnarValueException e) { diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/MergeColumnTupleReference.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/MergeColumnTupleReference.java index b14ea57..effd9f5 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/MergeColumnTupleReference.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/MergeColumnTupleReference.java @@ -18,8 +18,6 @@ */ package org.apache.asterix.column.tuple; -import java.util.BitSet; - import org.apache.asterix.column.bytes.stream.in.MultiByteBufferInputStream; import org.apache.asterix.column.operation.lsm.merge.IEndOfPageCallBack; import org.apache.asterix.column.operation.lsm.merge.MergeColumnReadMetadata; @@ -37,7 +35,6 @@ private final IColumnValuesReader[] columnReaders; private int skipCount; private IEndOfPageCallBack endOfPageCallBack; - private BitSet presentColumnIndexes; private int mergingLength; public MergeColumnTupleReference(int componentIndex, ColumnBTreeReadLeafFrame frame, @@ -90,16 +87,6 @@ } @Override - public void newPage() throws HyracksDataException { - super.newPage(); - frame.setPresentColumnsIndices(); - // the tuples are being read, meanwhile MegaLeaf changed - if (presentColumnIndexes != null) { - frame.getAllColumns(presentColumnIndexes); - } - } - - @Override protected void startColumnFilter(IColumnBufferProvider buffersProvider, int ordinal, int numberOfTuples) throws HyracksDataException { // NoOp @@ -155,12 +142,4 @@ } }; } - - public void setColumnIndexes(BitSet presentColumnsIndexes) { - this.presentColumnIndexes = presentColumnsIndexes; - } - - public void fillColumnIndexes() { - frame.getAllColumns(presentColumnIndexes); - } } diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java index 3623769..6d6deaa 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java @@ -249,7 +249,7 @@ if (numberOfColumns > zerothSegmentMaxColumns) { // read the rest of the columns from the segment stream currentColumnIndex = segmentBuffers.readOffset(offsetColumnIndexPairs, zerothSegmentMaxColumns, - maxNumberOfColumnsInAPage, currentColumnIndex); + maxNumberOfColumnsInAPage, currentColumnIndex, numberOfColumns); } return currentColumnIndex; } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/ColumnRanges.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/ColumnRanges.java index e6170a2..b41d8da 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/ColumnRanges.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/ColumnRanges.java @@ -23,6 +23,7 @@ import static org.apache.hyracks.storage.am.lsm.btree.column.utils.ColumnUtil.getColumnStartOffset; import static org.apache.hyracks.storage.am.lsm.btree.column.utils.ColumnUtil.getNumberOfRemainingPages; +import java.util.Arrays; import java.util.BitSet; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -143,6 +144,7 @@ // Get start page ID (given the computed length above) int startPageId = getColumnStartPageIndex(columnIndex); + // Get the number of pages (given the computed length above) int numberOfPages = getColumnNumberOfPages(columnIndex); @@ -253,9 +255,15 @@ return leafFrame.getMegaLeafNodeNumberOfPages(); } + public void pageZeroSegmentsInit(ColumnBTreeReadLeafFrame leafFrame) throws HyracksDataException { + this.leafFrame = leafFrame; + pageZeroId = leafFrame.getPageId(); + } + private void init() { int numberOfColumns = leafFrame.getNumberOfColumns(); offsetColumnIndexPairs = LongArrays.ensureCapacity(offsetColumnIndexPairs, numberOfColumns + 1, 0); + Arrays.fill(offsetColumnIndexPairs, 0, numberOfColumns + 1, Long.MAX_VALUE); lengths = IntArrays.ensureCapacity(lengths, numberOfColumns, 0); columnsOrder = IntArrays.ensureCapacity(columnsOrder, numberOfColumns + 1, 0); nonEvictablePages.clear(); diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java index bceec56..77845bf 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java @@ -137,6 +137,7 @@ return; } + columnRanges.pageZeroSegmentsInit(leafFrame); int pageZeroId = leafFrame.getPageId(); // Pinning all the segments of the page zero // as the column eviction logic is based on the length of the columns which diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudMegaPageReadContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudMegaPageReadContext.java index 1c42ec0..7baa203 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudMegaPageReadContext.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudMegaPageReadContext.java @@ -248,8 +248,8 @@ pageCounter++; } catch (Throwable th) { LOGGER.log(getLevel(th), - "Error while pinning page number {} with number of pages {}. stream: {}, columnRanges:\n {}", i, - numberOfPages, gapStream, columnRanges, th); + "Error while pinning page number {} having pageZeroId {} with number of pages {}. stream: {}, columnRanges:\n {}", + i, pageZeroId, numberOfPages, gapStream, columnRanges, th); throw th; } } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20151 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: ionic Gerrit-Change-Id: I1a5725e36301537a0926781d1d875ff62d523545 Gerrit-Change-Number: 20151 Gerrit-PatchSet: 7 Gerrit-Owner: Ritik Raj <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Peeyush Gupta <[email protected]> Gerrit-Reviewer: Ritik Raj <[email protected]> Gerrit-MessageType: merged
