This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit a31f0658a2a698b7b491c224c59a02e5caa1fddf Author: Ali Alsuliman <[email protected]> AuthorDate: Mon Feb 6 12:48:29 2023 -0800 [NO ISSUE][OTH] Add some logging when join build fails to insert tuple - user model changes: no - storage format changes: no - interface changes: no Change-Id: I5e342019d4831bfef0a10d6154a080c3cb524256 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17363 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> --- .../PreferToSpillFullyOccupiedFramePolicy.java | 10 ++++++++++ .../hyracks/dataflow/std/join/OptimizedHybridHashJoin.java | 14 ++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/PreferToSpillFullyOccupiedFramePolicy.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/PreferToSpillFullyOccupiedFramePolicy.java index 12985c0bec..613a396831 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/PreferToSpillFullyOccupiedFramePolicy.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/PreferToSpillFullyOccupiedFramePolicy.java @@ -98,4 +98,14 @@ public class PreferToSpillFullyOccupiedFramePolicy { } }; } + + public String partitionsStatus() { + StringBuilder sb = new StringBuilder(); + int numPartitions = bufferManager.getNumPartitions(); + for (int p = 0; p < numPartitions; p++) { + sb.append("p:").append(p).append(",#t:").append(bufferManager.getNumTuples(p)).append(",s:") + .append(spilledStatus.get(p)).append(",s:").append(bufferManager.getPhysicalSize(p)).append('\n'); + } + return sb.toString(); + } } diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java index 7a9bb25ff3..02a381de14 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java @@ -49,6 +49,8 @@ import org.apache.hyracks.dataflow.std.buffermanager.VPartitionTupleBufferManage import org.apache.hyracks.dataflow.std.structures.ISerializableTable; import org.apache.hyracks.dataflow.std.structures.SerializableHashTable; import org.apache.hyracks.dataflow.std.structures.TuplePointer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /** * This class mainly applies one level of HHJ on a pair of @@ -56,6 +58,7 @@ import org.apache.hyracks.dataflow.std.structures.TuplePointer; */ public class OptimizedHybridHashJoin { + private static final Logger LOGGER = LogManager.getLogger(); // Used for special probe BigObject which can not be held into the Join memory private FrameTupleAppender bigFrameAppender; @@ -155,12 +158,19 @@ public class OptimizedHybridHashJoin { int recordSize = VPartitionTupleBufferManager.calculateActualSize(null, accessorBuild.getTupleLength(tid)); double numFrames = (double) recordSize / (double) jobletCtx.getInitialFrameSize(); int victimPartition; - if (numFrames > bufferManager.getConstrain().frameLimit(pid) - || (victimPartition = spillPolicy.selectVictimPartition(pid)) < 0) { + int partitionFrameLimit = bufferManager.getConstrain().frameLimit(pid); + if (numFrames > partitionFrameLimit || (victimPartition = spillPolicy.selectVictimPartition(pid)) < 0) { // insert request can never be satisfied if (numFrames > memSizeInFrames || recordSize < jobletCtx.getInitialFrameSize()) { // the tuple is greater than the memory budget or although the record is small we could not find // a frame for it (possibly due to a bug) + String details = String.format( + "partition %s, tuple size %s, needed # frames %s, partition frame limit %s, join " + + "memory in frames %s, initial frame size %s", + pid, recordSize, numFrames, partitionFrameLimit, memSizeInFrames, + jobletCtx.getInitialFrameSize()); + LOGGER.debug("can't insert tuple in join memory. {}", details); + LOGGER.debug("partitions status:\n{}", spillPolicy.partitionsStatus()); throw HyracksDataException.create(ErrorCode.INSUFFICIENT_MEMORY); } // Record is large but insertion failed either 1) we could not satisfy the request because of the
