This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit f2ef282cc6396f84a8870b6045a1394c617a36d7
Author: Ritik Raj <raj.ritik9...@gmail.com>
AuthorDate: Wed Jun 18 01:53:11 2025 +0530
[ASTERIXDB-3601][STO] Properly unpinning segment files
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Properly unpinning the segment files, in case
of failure. Otherwise it might cause resource
leak.
Ext-ref: MB-66306
Change-Id: I1dec880871008fc6b9be6662df9fa034992858c9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19990
Reviewed-by: Peeyush Gupta <peeyush.gu...@couchbase.com>
Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Tested-by: Ritik Raj <ritik....@couchbase.com>
---
.../src/test/resources/cc-single-partition.conf | 2 +-
...MultiPersistentPageZeroBufferBytesOutputStream.java | 2 +-
.../column/zero/PageZeroWriterFlavorSelector.java | 2 +-
.../zero/readers/DefaultColumnPageZeroReader.java | 18 ++++++++++++++++++
.../multipage/AbstractColumnMultiPageZeroReader.java | 3 +++
.../multipage/DefaultColumnMultiPageZeroReader.java | 14 ++++++++++++++
.../multipage/SparseColumnMultiPageZeroReader.java | 15 +++++++++++++++
.../am/lsm/btree/column/cloud/sweep/ColumnSweeper.java | 16 +++++++++++++++-
.../column/impls/btree/ColumnBTreeReadLeafFrame.java | 4 ++++
.../column/impls/btree/IColumnPageZeroReader.java | 2 ++
10 files changed, 74 insertions(+), 4 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf
b/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf
index d5837adb17..23071d6501 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf
@@ -53,4 +53,4 @@ compiler.windowmemory=192KB
compiler.ordered.fields=false
messaging.frame.size=4096
messaging.frame.count=512
-storage.buffercache.pagesize=32KB
+storage.buffercache.pagesize=32KB
\ No newline at end of file
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentPageZeroBufferBytesOutputStream.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentPageZeroBufferBytesOutputStream.java
index c156e49d2d..4ae01536c7 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentPageZeroBufferBytesOutputStream.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentPageZeroBufferBytesOutputStream.java
@@ -42,7 +42,7 @@ public final class
MultiPersistentPageZeroBufferBytesOutputStream extends Abstra
@Override
protected void preReset() {
- if (allocatedBytes > 0) {
+ if (allocatedBytes > 0 || !buffers.isEmpty()) {
//This should not be the case, with the pageZero segments.
//As the stream should be finished after the flush.
throw new IllegalStateException("PageZero segments should already
be finished after flush");
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/PageZeroWriterFlavorSelector.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/PageZeroWriterFlavorSelector.java
index 06e441aa78..831e459ba6 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/PageZeroWriterFlavorSelector.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/PageZeroWriterFlavorSelector.java
@@ -44,12 +44,12 @@ import it.unimi.dsi.fastutil.bytes.Byte2ObjectArrayMap;
* This class implements an optimization strategy for sparse columns:
* - Default writer: Allocates space for all columns in the schema (suitable
for dense data)
* - Sparse writer: Only allocates space for present columns (suitable for
sparse data)
+ * - Multi-page writers: Variants of the above writers that support multi-page
operations
*</p>
* The selector automatically chooses the most space-efficient option based on
the actual
* space requirements of each approach.
*/
public class PageZeroWriterFlavorSelector implements
IColumnPageZeroWriterFlavorSelector {
- // Flag indicating which writer type is currently selected
(DEFAULT_WRITER_FLAG=default, SPARSE_WRITER_FLAG=sparse)
protected byte writerFlag =
IColumnPageZeroWriter.ColumnPageZeroWriterType.ADAPTIVE.getWriterFlag();
// Cache of writer instances to avoid repeated object creation
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/DefaultColumnPageZeroReader.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/DefaultColumnPageZeroReader.java
index 1692e70d67..a34d8c1af1 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/DefaultColumnPageZeroReader.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/DefaultColumnPageZeroReader.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.column.zero.readers;
+import static
org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.FLAG_OFFSET;
import static
org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.LEFT_MOST_KEY_OFFSET;
import static
org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.MEGA_LEAF_NODE_LENGTH;
import static
org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.NEXT_LEAF_OFFSET;
@@ -31,9 +32,16 @@ import java.util.BitSet;
import org.apache.asterix.column.zero.writers.DefaultColumnPageZeroWriter;
import
org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
import org.apache.hyracks.storage.am.lsm.btree.column.cloud.IntPairUtil;
+import
org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException;
import
org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroReader;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
public class DefaultColumnPageZeroReader implements IColumnPageZeroReader {
+ protected static Logger LOGGER = LogManager.getLogger();
+
protected ByteBuffer pageZeroBuf;
protected BitSet pageZeroSegmentsPages;
protected int numberOfPresentColumns;
@@ -182,4 +190,14 @@ public class DefaultColumnPageZeroReader implements
IColumnPageZeroReader {
pageZeroSegmentsPages.set(0);
return pageZeroSegmentsPages;
}
+
+ @Override
+ public void printPageZeroReaderInfo() {
+ ColumnarValueException ex = new ColumnarValueException();
+ ObjectNode readerNode = ex.createNode(getClass().getSimpleName());
+ readerNode.put("headerSize", headerSize);
+ readerNode.put("numberOfPresentColumns", numberOfPresentColumns);
+ readerNode.put("flag", pageZeroBuf.get(FLAG_OFFSET));
+ LOGGER.debug("SingleColumnPageZeroReader Info: {}",
readerNode.toPrettyString());
+ }
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/AbstractColumnMultiPageZeroReader.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/AbstractColumnMultiPageZeroReader.java
index 3b05fc38a8..b0fc721baa 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/AbstractColumnMultiPageZeroReader.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/AbstractColumnMultiPageZeroReader.java
@@ -20,8 +20,11 @@ package org.apache.asterix.column.zero.writers.multipage;
import
org.apache.asterix.column.bytes.stream.in.MultiPageZeroByteBuffersReader;
import
org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroReader;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public abstract class AbstractColumnMultiPageZeroReader implements
IColumnPageZeroReader {
+ protected static final Logger LOGGER = LogManager.getLogger();
protected MultiPageZeroByteBuffersReader segmentBuffers;
AbstractColumnMultiPageZeroReader() {
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java
index db800ae877..5a0b1808d3 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java
@@ -40,6 +40,9 @@ import org.apache.hyracks.data.std.primitive.LongPointable;
import org.apache.hyracks.data.std.primitive.VoidPointable;
import
org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
import org.apache.hyracks.storage.am.lsm.btree.column.cloud.IntPairUtil;
+import
org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
public class DefaultColumnMultiPageZeroReader extends
AbstractColumnMultiPageZeroReader {
public static final int headerSize =
DefaultColumnMultiPageZeroWriter.EXTENDED_HEADER_SIZE;
@@ -273,4 +276,15 @@ public class DefaultColumnMultiPageZeroReader extends
AbstractColumnMultiPageZer
public int getHeaderSize() {
return EXTENDED_HEADER_SIZE;
}
+
+ @Override
+ public void printPageZeroReaderInfo() {
+ ColumnarValueException ex = new ColumnarValueException();
+ ObjectNode readerNode = ex.createNode(getClass().getSimpleName());
+ readerNode.put("headerSize", headerSize);
+ readerNode.put("maxColumnsInZerothSegment", zerothSegmentMaxColumns);
+ readerNode.put("maxNumberOfColumnsInAPage", maxNumberOfColumnsInAPage);
+ readerNode.put("numberOfPageZeroSegments", numberOfPageZeroSegments);
+ LOGGER.debug("DefaultColumnMultiPageZeroReader Info: {}",
readerNode.toPrettyString());
+ }
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroReader.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroReader.java
index d555bdc88d..0a0a817048 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroReader.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroReader.java
@@ -40,6 +40,9 @@ import org.apache.hyracks.data.std.primitive.LongPointable;
import org.apache.hyracks.data.std.primitive.VoidPointable;
import
org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
import org.apache.hyracks.storage.am.lsm.btree.column.cloud.IntPairUtil;
+import
org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
@@ -358,4 +361,16 @@ public class SparseColumnMultiPageZeroReader extends
AbstractColumnMultiPageZero
}
return pageZeroSegmentsPages;
}
+
+ @Override
+ public void printPageZeroReaderInfo() {
+ ColumnarValueException ex = new ColumnarValueException();
+ ObjectNode readerNode = ex.createNode(getClass().getSimpleName());
+ readerNode.put("headerSize", headerSize);
+ readerNode.put("maxColumnIndexInZerothSegment",
maxColumnIndexInZerothSegment);
+ readerNode.put("numberOfColumnInZerothSegment",
numberOfColumnInZerothSegment);
+ readerNode.put("maxNumberOfColumnsInAPage", maxNumberOfColumnsInAPage);
+ readerNode.put("numberOfPageZeroSegments", numberOfPageZeroSegments);
+ LOGGER.debug("SparseColumnMultiPageZeroReader Info: {}",
readerNode.toPrettyString());
+ }
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java
index 1f0c7f4d6c..ba4376f963 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java
@@ -165,6 +165,7 @@ public final class ColumnSweeper {
int freedSpace = 0;
context.open(fileId);
try {
+ Throwable failure = null;
while (nextPageId >= 0) {
if (context.stopSweeping()) {
// Exit as the index is being dropped
@@ -183,17 +184,30 @@ public final class ColumnSweeper {
ranges.reset(leafFrame, plan);
freedSpace += punchHoles(context, leafFrame);
}
+ } catch (Throwable th) {
+ failure = th;
} finally {
if (columnsLocked) {
page0.sweepUnlock();
}
// unpin segment pages
for (ICachedPage cachedPage : segmentPagesTempHolder) {
- context.unpin(cachedPage, bcOpCtx);
+ try {
+ context.unpin(cachedPage, bcOpCtx);
+ } catch (Throwable e) {
+ if (failure == null) {
+ failure = e;
+ } else {
+ failure.addSuppressed(e);
+ }
+ }
}
context.unpin(page0, bcOpCtx);
}
}
+ if (failure != null) {
+ throw HyracksDataException.create(failure);
+ }
} finally {
context.close();
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java
index 4a784000a1..c3f822815f 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java
@@ -188,4 +188,8 @@ public final class ColumnBTreeReadLeafFrame extends
AbstractColumnBTreeLeafFrame
public BitSet markRequiredPageZeroSegments(BitSet projectedColumns, int
pageZeroId, boolean markAll) {
return columnPageZeroReader.markRequiredPageSegments(projectedColumns,
pageZeroId, markAll);
}
+
+ public void printPageZeroReaderInfo() {
+ columnPageZeroReader.printPageZeroReaderInfo();
+ }
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroReader.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroReader.java
index db4f778c95..d9cf11be9f 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroReader.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroReader.java
@@ -75,4 +75,6 @@ public interface IColumnPageZeroReader {
void resetStream(IColumnBufferProvider pageZeroSegmentBufferProvider)
throws HyracksDataException;
BitSet markRequiredPageSegments(BitSet projectedColumns, int pageZeroId,
boolean markAll);
+
+ void printPageZeroReaderInfo();
}
