>From Ritik Raj <[email protected]>: Ritik Raj has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19647 )
Change subject: [ASTERIXDB-3597][STO] Add debug lines for pageZero flush failure ...................................................................... [ASTERIXDB-3597][STO] Add debug lines for pageZero flush failure - user model changes: no - storage format changes: no - interface changes: no Ext-ref: MB-66227 Change-Id: I81710a257cc14383ade9e34ea723e0fa15d7a736 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19647 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Peeyush Gupta <[email protected]> --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeWriteLeafFrame.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleWriter.java M asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/bytes/AbstractBytesTest.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java 4 files changed, 77 insertions(+), 7 deletions(-) Approvals: Peeyush Gupta: Looks good to me, approved Jenkins: Verified; Verified Anon. E. Moose #1000171: diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/bytes/AbstractBytesTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/bytes/AbstractBytesTest.java index 8fd9f6b..9d57dde 100644 --- a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/bytes/AbstractBytesTest.java +++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/bytes/AbstractBytesTest.java @@ -209,7 +209,7 @@ //Reserved for the number of pages int requiredFreeSpace = HEADER_SIZE; //Columns' Offsets - requiredFreeSpace += columnWriter.getColumnOffsetsSize(); + requiredFreeSpace += columnWriter.getColumnOffsetsSize(true); //Occupied space from previous writes requiredFreeSpace += columnWriter.getOccupiedSpace(); //New tuple required space diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleWriter.java index 92b3892..14f2399 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleWriter.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleWriter.java @@ -62,8 +62,8 @@ * * @return the size needed to store columns' offsets */ - public final int getColumnOffsetsSize() { - return Integer.BYTES * getNumberOfColumns(true); + public final int getColumnOffsetsSize(boolean includeCurrentTupleColumns) { + return Integer.BYTES * getNumberOfColumns(includeCurrentTupleColumns); } /** diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java index ee61fa6..114ac05 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java @@ -40,9 +40,12 @@ import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback; import org.apache.hyracks.storage.common.buffercache.context.IBufferCacheWriteContext; import org.apache.hyracks.storage.common.file.BufferedFileHandle; +import org.apache.hyracks.util.JSONUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.fasterxml.jackson.databind.node.ObjectNode; + public final class ColumnBTreeBulkloader extends BTreeNSMBulkLoader implements IColumnWriteMultiPageOp { private static final Logger LOGGER = LogManager.getLogger(); private final List<CachedPage> columnsPages; @@ -60,6 +63,7 @@ private int maxNumberOfPagesForAColumn; private int maxNumberOfPagesInALeafNode; private int maxTupleCount; + private int lastRequiredFreeSpace; public ColumnBTreeBulkloader(float fillFactor, boolean verifyInput, IPageWriteCallback callback, ITreeIndex index, ITreeIndexFrame leafFrame, IBufferCacheWriteContext writeContext) throws HyracksDataException { @@ -80,6 +84,7 @@ maxNumberOfPagesInALeafNode = 0; numberOfLeafNodes = 1; maxTupleCount = 0; + lastRequiredFreeSpace = 0; } @Override @@ -109,13 +114,14 @@ int requiredFreeSpace = AbstractColumnBTreeLeafFrame.HEADER_SIZE; //Columns' Offsets columnWriter.updateColumnMetadataForCurrentTuple(tuple); - requiredFreeSpace += columnWriter.getColumnOffsetsSize(); + requiredFreeSpace += columnWriter.getColumnOffsetsSize(true); //Occupied space from previous writes requiredFreeSpace += columnWriter.getOccupiedSpace(); //min and max tuples' sizes requiredFreeSpace += lowKey.getTuple().getTupleSize() + getSplitKeySize(tuple); //New tuple required space requiredFreeSpace += columnWriter.bytesRequired(tuple); + lastRequiredFreeSpace = requiredFreeSpace; return bufferCache.getPageSize() <= requiredFreeSpace; } @@ -132,7 +138,12 @@ public void end() throws HyracksDataException { if (tupleCount > 0) { splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer().array(), 0); - columnarFrame.flush(columnWriter, tupleCount, lowKey.getTuple(), splitKey.getTuple()); + try { + columnarFrame.flush(columnWriter, tupleCount, lowKey.getTuple(), splitKey.getTuple()); + } catch (Exception e) { + logState(e); + throw e; + } } columnWriter.close(); //We are done, return any temporary confiscated pages @@ -156,7 +167,12 @@ splitKey.setLeftPage(leafFrontier.pageId); if (tupleCount > 0) { //We need to flush columns to confiscate all columns pages first before calling propagateBulk - columnarFrame.flush(columnWriter, tupleCount, lowKey.getTuple(), splitKey.getTuple()); + try { + columnarFrame.flush(columnWriter, tupleCount, lowKey.getTuple(), splitKey.getTuple()); + } catch (Exception e) { + logState(e); + throw e; + } } propagateBulk(1, pagesToWrite); @@ -285,4 +301,26 @@ tempConfiscatedPages.add(page); return page.getBuffer(); } + + private void logState(Exception e) { + try { + ObjectNode state = JSONUtil.createObject(); + // number of tuples processed for the current leaf + state.put("currentLeafTupleCount", tupleCount); + // number of columns + state.put("currentLeafColumnCount", columnWriter.getNumberOfColumns(false)); + // number of columns including current tuple + state.put("currentColumnCount", columnWriter.getNumberOfColumns(true)); + state.put("lastRequiredFreeSpace", lastRequiredFreeSpace); + state.put("splitKeyTupleSize", splitKey.getTuple().getTupleSize()); + state.put("splitKeyTupleSizeByTupleWriter", tupleWriter.bytesRequired(splitKey.getTuple())); + state.put("lowKeyTupleSize", lowKey.getTuple().getTupleSize()); + ObjectNode bufNode = state.putObject("leafBufferDetails"); + columnarFrame.dumpBuffer(bufNode); + LOGGER.error("pageZero flush failed {}", state, e); + } catch (Exception ex) { + e.addSuppressed(ex); + } + } + } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeWriteLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeWriteLeafFrame.java index bced7ef..1fc5712 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeWriteLeafFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeWriteLeafFrame.java @@ -23,6 +23,8 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter; +import com.fasterxml.jackson.databind.node.ObjectNode; + public class ColumnBTreeWriteLeafFrame extends AbstractColumnBTreeLeafFrame { private final AbstractColumnTupleWriter columnTupleWriter; @@ -63,7 +65,7 @@ int numberOfColumns = columnWriter.getNumberOfColumns(false); buf.putInt(TUPLE_COUNT_OFFSET, numberOfTuples); buf.putInt(NUMBER_OF_COLUMNS_OFFSET, numberOfColumns); - buf.putInt(SIZE_OF_COLUMNS_OFFSETS_OFFSET, columnWriter.getColumnOffsetsSize()); + buf.putInt(SIZE_OF_COLUMNS_OFFSETS_OFFSET, columnWriter.getColumnOffsetsSize(false)); } public AbstractColumnTupleWriter getColumnTupleWriter() { @@ -73,4 +75,16 @@ void setNextLeaf(int pageId) { buf.putInt(NEXT_LEAF_OFFSET, pageId); } + + public void dumpBuffer(ObjectNode bufNode) { + bufNode.put("tupleCount", buf.getInt(TUPLE_COUNT_OFFSET)); + bufNode.put("level", buf.get(Constants.LEVEL_OFFSET)); + bufNode.put("numberOfColumns", buf.getInt(NUMBER_OF_COLUMNS_OFFSET)); + bufNode.put("leftMostKeyOffset", buf.getInt(LEFT_MOST_KEY_OFFSET)); + bufNode.put("rightMostKeyOffset", buf.getInt(RIGHT_MOST_KEY_OFFSET)); + bufNode.put("sizeOfColumns", buf.getInt(SIZE_OF_COLUMNS_OFFSETS_OFFSET)); + bufNode.put("megaLeafNodeLength", buf.getInt(MEGA_LEAF_NODE_LENGTH)); + bufNode.put("flagOffset", buf.get(FLAG_OFFSET)); + bufNode.put("nextLeafOffset", buf.getInt(NEXT_LEAF_OFFSET)); + } } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19647 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: ionic Gerrit-Change-Id: I81710a257cc14383ade9e34ea723e0fa15d7a736 Gerrit-Change-Number: 19647 Gerrit-PatchSet: 2 Gerrit-Owner: Ritik Raj <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Peeyush Gupta <[email protected]> Gerrit-Reviewer: Ritik Raj <[email protected]> Gerrit-MessageType: merged
