This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit b1ae88e7e6bae81b6e9de5932ea16c8ffd077320 Author: Murtadha Hubail <[email protected]> AuthorDate: Sat Oct 31 23:35:56 2020 +0300 [NO ISSUE][STO] Log bulkloader state on failure - user model changes: no - storage format changes: no - interface changes: yes Details: - Log BTree bulkloader state when an exception is encountered. Change-Id: I00a69d4883ae7a69c28a201408504049dd70ffb5 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/8683 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> --- .../hyracks/hyracks-storage-am-btree/pom.xml | 13 +++++++++++ .../hyracks/storage/am/btree/impls/BTree.java | 26 +++++++++++++++++++++- .../hyracks/hyracks-storage-am-common/pom.xml | 4 ++++ .../storage/am/common/api/ITreeIndexFrame.java | 13 +++++++++++ .../am/common/frames/TreeIndexNSMFrame.java | 20 +++++++++++++++++ 5 files changed, 75 insertions(+), 1 deletion(-) diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml index fa74184..45330e8 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml @@ -107,5 +107,18 @@ <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-util</artifactId> + <version>${project.version}</version> + </dependency> </dependencies> </project> \ No newline at end of file diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java index 3a062af..f62f2d1 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java @@ -69,11 +69,16 @@ import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback; import org.apache.hyracks.storage.common.file.BufferedFileHandle; +import org.apache.hyracks.util.JSONUtil; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.fasterxml.jackson.databind.node.ObjectNode; public class BTree extends AbstractTreeIndex { public static final float DEFAULT_FILL_FACTOR = 0.7f; - + private static final Logger LOGGER = LogManager.getLogger(); private static final long RESTART_OP = Long.MIN_VALUE; private static final long FULL_RESTART_OP = Long.MIN_VALUE + 1; private static final int MAX_RESTARTS = 10; @@ -1086,6 +1091,7 @@ public class BTree extends AbstractTreeIndex { } ((IBTreeLeafFrame) leafFrame).insertSorted(tuple); } catch (HyracksDataException | RuntimeException e) { + logState(tuple, e); handleException(); throw e; } @@ -1194,6 +1200,24 @@ public class BTree extends AbstractTreeIndex { public void abort() throws HyracksDataException { super.handleException(); } + + private void logState(ITupleReference tuple, Exception e) { + try { + ObjectNode state = JSONUtil.createObject(); + state.set("leafFrame", leafFrame.getState()); + state.set("interiorFrame", interiorFrame.getState()); + int tupleSize = Math.max(leafFrame.getBytesRequiredToWriteTuple(tuple), + interiorFrame.getBytesRequiredToWriteTuple(tuple)); + state.put("tupleSize", tupleSize); + state.put("spaceNeeded", tupleWriter.bytesRequired(tuple) + slotSize); + state.put("spaceUsed", leafFrame.getBuffer().capacity() - leafFrame.getTotalFreeSpace()); + state.put("leafMaxBytes", leafMaxBytes); + state.put("maxTupleSize", maxTupleSize); + LOGGER.error("failed to add tuple {}", state, e); + } catch (Throwable t) { + e.addSuppressed(t); + } + } } @SuppressWarnings("rawtypes") diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-common/pom.xml index 31463b7..703deb8 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/pom.xml @@ -109,5 +109,9 @@ <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </dependency> </dependencies> </project> \ No newline at end of file diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java index dc59612..18d5653 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java @@ -28,6 +28,9 @@ import org.apache.hyracks.storage.common.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; +import org.apache.hyracks.util.JSONUtil; + +import com.fasterxml.jackson.databind.node.ObjectNode; public interface ITreeIndexFrame { @@ -122,4 +125,14 @@ public interface ITreeIndexFrame { public ITupleReference getLeftmostTuple() throws HyracksDataException; public ITupleReference getRightmostTuple() throws HyracksDataException; + + default ObjectNode getState() { + ObjectNode json = JSONUtil.createObject(); + json.put("tupleCount", getTupleCount()); + json.put("freeSpaceOff", getFreeSpaceOff()); + json.put("level", getLevel()); + json.put("pageLsn", getPageLsn()); + json.put("totalFreeSpace", getTotalFreeSpace()); + return json; + } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java index 6106358..08d4564 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java @@ -31,6 +31,10 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; import org.apache.hyracks.storage.am.common.ophelpers.SlotOffTupleOff; import org.apache.hyracks.storage.common.buffercache.ICachedPage; +import org.apache.hyracks.util.JSONUtil; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; public abstract class TreeIndexNSMFrame implements ITreeIndexFrame { @@ -354,4 +358,20 @@ public abstract class TreeIndexNSMFrame implements ITreeIndexFrame { return frameTuple; } } + + @Override + public ObjectNode getState() { + ObjectNode state = ITreeIndexFrame.super.getState(); + state.put("largeFlag", getLargeFlag()); + return state; + } + + @Override + public String toString() { + try { + return JSONUtil.convertNode(getState()); + } catch (JsonProcessingException e) { + return "failed to convert json"; + } + } }
