This is an automated email from the ASF dual-hosted git repository. alsuliman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit fa27f73178b87efee78971894fedcebb15e7b64d Author: Ritik Raj <[email protected]> AuthorDate: Wed Apr 16 21:36:25 2025 +0530 [ASTERIXDB-3597][STO] Add debug lines for pageZero flush failure - user model changes: no - storage format changes: no - interface changes: no Ext-ref: MB-66227 Change-Id: I81710a257cc14383ade9e34ea723e0fa15d7a736 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19647 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Peeyush Gupta <[email protected]> --- .../column/test/bytes/AbstractBytesTest.java | 2 +- .../column/api/AbstractColumnTupleWriter.java | 4 +- .../column/impls/btree/ColumnBTreeBulkloader.java | 44 ++++++++++++++++++++-- .../impls/btree/ColumnBTreeWriteLeafFrame.java | 16 +++++++- 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/bytes/AbstractBytesTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/bytes/AbstractBytesTest.java index 8fd9f6b890..9d57ddea9e 100644 --- a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/bytes/AbstractBytesTest.java +++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/bytes/AbstractBytesTest.java @@ -209,7 +209,7 @@ public abstract class AbstractBytesTest extends TestBase { //Reserved for the number of pages int requiredFreeSpace = HEADER_SIZE; //Columns' Offsets - requiredFreeSpace += columnWriter.getColumnOffsetsSize(); + requiredFreeSpace += columnWriter.getColumnOffsetsSize(true); //Occupied space from previous writes requiredFreeSpace += columnWriter.getOccupiedSpace(); //New tuple required space diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleWriter.java index 92b3892b68..14f239919b 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleWriter.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/AbstractColumnTupleWriter.java @@ -62,8 +62,8 @@ public abstract class AbstractColumnTupleWriter extends AbstractTupleWriterDisab * * @return the size needed to store columns' offsets */ - public final int getColumnOffsetsSize() { - return Integer.BYTES * getNumberOfColumns(true); + public final int getColumnOffsetsSize(boolean includeCurrentTupleColumns) { + return Integer.BYTES * getNumberOfColumns(includeCurrentTupleColumns); } /** diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java index ee61fa6f8e..114ac05978 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java @@ -40,9 +40,12 @@ import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback; import org.apache.hyracks.storage.common.buffercache.context.IBufferCacheWriteContext; import org.apache.hyracks.storage.common.file.BufferedFileHandle; +import org.apache.hyracks.util.JSONUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.fasterxml.jackson.databind.node.ObjectNode; + public final class ColumnBTreeBulkloader extends BTreeNSMBulkLoader implements IColumnWriteMultiPageOp { private static final Logger LOGGER = LogManager.getLogger(); private final List<CachedPage> columnsPages; @@ -60,6 +63,7 @@ public final class ColumnBTreeBulkloader extends BTreeNSMBulkLoader implements I private int maxNumberOfPagesForAColumn; private int maxNumberOfPagesInALeafNode; private int maxTupleCount; + private int lastRequiredFreeSpace; public ColumnBTreeBulkloader(float fillFactor, boolean verifyInput, IPageWriteCallback callback, ITreeIndex index, ITreeIndexFrame leafFrame, IBufferCacheWriteContext writeContext) throws HyracksDataException { @@ -80,6 +84,7 @@ public final class ColumnBTreeBulkloader extends BTreeNSMBulkLoader implements I maxNumberOfPagesInALeafNode = 0; numberOfLeafNodes = 1; maxTupleCount = 0; + lastRequiredFreeSpace = 0; } @Override @@ -109,13 +114,14 @@ public final class ColumnBTreeBulkloader extends BTreeNSMBulkLoader implements I int requiredFreeSpace = AbstractColumnBTreeLeafFrame.HEADER_SIZE; //Columns' Offsets columnWriter.updateColumnMetadataForCurrentTuple(tuple); - requiredFreeSpace += columnWriter.getColumnOffsetsSize(); + requiredFreeSpace += columnWriter.getColumnOffsetsSize(true); //Occupied space from previous writes requiredFreeSpace += columnWriter.getOccupiedSpace(); //min and max tuples' sizes requiredFreeSpace += lowKey.getTuple().getTupleSize() + getSplitKeySize(tuple); //New tuple required space requiredFreeSpace += columnWriter.bytesRequired(tuple); + lastRequiredFreeSpace = requiredFreeSpace; return bufferCache.getPageSize() <= requiredFreeSpace; } @@ -132,7 +138,12 @@ public final class ColumnBTreeBulkloader extends BTreeNSMBulkLoader implements I public void end() throws HyracksDataException { if (tupleCount > 0) { splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer().array(), 0); - columnarFrame.flush(columnWriter, tupleCount, lowKey.getTuple(), splitKey.getTuple()); + try { + columnarFrame.flush(columnWriter, tupleCount, lowKey.getTuple(), splitKey.getTuple()); + } catch (Exception e) { + logState(e); + throw e; + } } columnWriter.close(); //We are done, return any temporary confiscated pages @@ -156,7 +167,12 @@ public final class ColumnBTreeBulkloader extends BTreeNSMBulkLoader implements I splitKey.setLeftPage(leafFrontier.pageId); if (tupleCount > 0) { //We need to flush columns to confiscate all columns pages first before calling propagateBulk - columnarFrame.flush(columnWriter, tupleCount, lowKey.getTuple(), splitKey.getTuple()); + try { + columnarFrame.flush(columnWriter, tupleCount, lowKey.getTuple(), splitKey.getTuple()); + } catch (Exception e) { + logState(e); + throw e; + } } propagateBulk(1, pagesToWrite); @@ -285,4 +301,26 @@ public final class ColumnBTreeBulkloader extends BTreeNSMBulkLoader implements I tempConfiscatedPages.add(page); return page.getBuffer(); } + + private void logState(Exception e) { + try { + ObjectNode state = JSONUtil.createObject(); + // number of tuples processed for the current leaf + state.put("currentLeafTupleCount", tupleCount); + // number of columns + state.put("currentLeafColumnCount", columnWriter.getNumberOfColumns(false)); + // number of columns including current tuple + state.put("currentColumnCount", columnWriter.getNumberOfColumns(true)); + state.put("lastRequiredFreeSpace", lastRequiredFreeSpace); + state.put("splitKeyTupleSize", splitKey.getTuple().getTupleSize()); + state.put("splitKeyTupleSizeByTupleWriter", tupleWriter.bytesRequired(splitKey.getTuple())); + state.put("lowKeyTupleSize", lowKey.getTuple().getTupleSize()); + ObjectNode bufNode = state.putObject("leafBufferDetails"); + columnarFrame.dumpBuffer(bufNode); + LOGGER.error("pageZero flush failed {}", state, e); + } catch (Exception ex) { + e.addSuppressed(ex); + } + } + } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeWriteLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeWriteLeafFrame.java index bced7efd10..1fc5712112 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeWriteLeafFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeWriteLeafFrame.java @@ -23,6 +23,8 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleWriter; +import com.fasterxml.jackson.databind.node.ObjectNode; + public class ColumnBTreeWriteLeafFrame extends AbstractColumnBTreeLeafFrame { private final AbstractColumnTupleWriter columnTupleWriter; @@ -63,7 +65,7 @@ public class ColumnBTreeWriteLeafFrame extends AbstractColumnBTreeLeafFrame { int numberOfColumns = columnWriter.getNumberOfColumns(false); buf.putInt(TUPLE_COUNT_OFFSET, numberOfTuples); buf.putInt(NUMBER_OF_COLUMNS_OFFSET, numberOfColumns); - buf.putInt(SIZE_OF_COLUMNS_OFFSETS_OFFSET, columnWriter.getColumnOffsetsSize()); + buf.putInt(SIZE_OF_COLUMNS_OFFSETS_OFFSET, columnWriter.getColumnOffsetsSize(false)); } public AbstractColumnTupleWriter getColumnTupleWriter() { @@ -73,4 +75,16 @@ public class ColumnBTreeWriteLeafFrame extends AbstractColumnBTreeLeafFrame { void setNextLeaf(int pageId) { buf.putInt(NEXT_LEAF_OFFSET, pageId); } + + public void dumpBuffer(ObjectNode bufNode) { + bufNode.put("tupleCount", buf.getInt(TUPLE_COUNT_OFFSET)); + bufNode.put("level", buf.get(Constants.LEVEL_OFFSET)); + bufNode.put("numberOfColumns", buf.getInt(NUMBER_OF_COLUMNS_OFFSET)); + bufNode.put("leftMostKeyOffset", buf.getInt(LEFT_MOST_KEY_OFFSET)); + bufNode.put("rightMostKeyOffset", buf.getInt(RIGHT_MOST_KEY_OFFSET)); + bufNode.put("sizeOfColumns", buf.getInt(SIZE_OF_COLUMNS_OFFSETS_OFFSET)); + bufNode.put("megaLeafNodeLength", buf.getInt(MEGA_LEAF_NODE_LENGTH)); + bufNode.put("flagOffset", buf.get(FLAG_OFFSET)); + bufNode.put("nextLeafOffset", buf.getInt(NEXT_LEAF_OFFSET)); + } }
