Forced synchronization on method that still occasionally fails multi-threaded test. This is a fix for a very rare multithreading issue we've observed in Joshua. We have a test that is able to reproduce the error fairly often when run on a host with multiple physical cores. This patch fixes all errors seen in both the patch and during runtime.
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/cadd987c Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/cadd987c Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/cadd987c Branch: refs/heads/master Commit: cadd987c16ff012298b42074fb96bab8697fa84f Parents: cabb52c Author: Kellen Sunderland <[email protected]> Authored: Tue Mar 29 15:46:27 2016 +0200 Committer: Kellen Sunderland <[email protected]> Committed: Thu Mar 31 10:44:43 2016 +0200 ---------------------------------------------------------------------- .../decoder/ff/tm/packed/PackedGrammar.java | 26 +++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/cadd987c/src/joshua/decoder/ff/tm/packed/PackedGrammar.java ---------------------------------------------------------------------- diff --git a/src/joshua/decoder/ff/tm/packed/PackedGrammar.java b/src/joshua/decoder/ff/tm/packed/PackedGrammar.java index 2251f5a..18aa60e 100644 --- a/src/joshua/decoder/ff/tm/packed/PackedGrammar.java +++ b/src/joshua/decoder/ff/tm/packed/PackedGrammar.java @@ -46,6 +46,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; +import java.nio.BufferUnderflowException; import java.nio.IntBuffer; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; @@ -487,6 +488,7 @@ public class PackedGrammar extends AbstractGrammar { feature_position += EncoderConfiguration.ID_SIZE; StringBuilder sb = new StringBuilder(); + for (int i = 0; i < num_features; i++) { int feature_id = encoding.readId(features, feature_position); FloatEncoder encoder = encoding.encoder(feature_id); @@ -505,18 +507,34 @@ public class PackedGrammar extends AbstractGrammar { return sb.toString().trim(); } - private final byte[] getAlignmentArray(int block_id) { + /** + * We need to synchronize this method as there is a many to one ratio between + * PackedRule/PhrasePair and this class (PackedSlice). This means during concurrent first + * getAlignments calls to PackedRule objects they could alter each other's positions within the + * buffer before calling read on the buffer. + */ + private synchronized final byte[] getAlignmentArray(int block_id) { if (alignments == null) throw new RuntimeException("No alignments available."); int alignment_position = alignmentLookup[block_id]; int num_points = (int) alignments.get(alignment_position); byte[] alignment = new byte[num_points * 2]; - + alignments.position(alignment_position + 1); - alignments.get(alignment, 0, num_points * 2); + try { + alignments.get(alignment, 0, num_points * 2); + } catch (BufferUnderflowException bue) { + Decoder.LOG(4, "Had an exception when accessing alignment mapped byte buffer"); + Decoder.LOG(4, "Attempting to access alignments at position: " + alignment_position + 1); + Decoder.LOG(4, "And to read this many bytes: " + num_points * 2); + Decoder.LOG(4, "Buffer capacity is : " + alignments.capacity()); + Decoder.LOG(4, "Buffer position is : " + alignments.position()); + Decoder.LOG(4, "Buffer limit is : " + alignments.limit()); + throw bue; + } return alignment; } - + private final PackedTrie root() { return getTrie(0); }
