PHOENIX-2649 - GC/OOM during BulkLoad (Sergey Soldatov)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a82a0ff6 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a82a0ff6 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a82a0ff6 Branch: refs/heads/calcite Commit: a82a0ff608f90c470a1290296f8ed08243956fc9 Parents: 7972422 Author: ravimagham <ravi.mag...@bazaarvoice.com> Authored: Thu Feb 4 14:38:43 2016 -0800 Committer: ravimagham <ravi.mag...@bazaarvoice.com> Committed: Thu Feb 4 14:38:43 2016 -0800 ---------------------------------------------------------------------- .../mapreduce/bulkload/TableRowkeyPair.java | 38 +++++++++++++++++--- .../mapreduce/bulkload/TestTableRowkeyPair.java | 6 ++-- 2 files changed, 37 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/a82a0ff6/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java index e3032be..ac80341 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java @@ -22,7 +22,7 @@ import java.io.DataOutput; import java.io.IOException; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.io.WritableUtils; @@ -30,6 +30,7 @@ import org.apache.hadoop.io.WritableUtils; import com.google.common.base.Preconditions; + /** * A WritableComparable to hold the table name and the rowkey. */ @@ -101,9 +102,38 @@ public class TableRowkeyPair implements WritableComparable<TableRowkeyPair> { return this.tableName.compareTo(otherTableName); } } - - static { - WritableComparator.define(TableRowkeyPair.class, new BytesWritable.Comparator()); + + /** Comparator for <code>TableRowkeyPair</code>. */ + public static class Comparator extends WritableComparator { + + public Comparator() { + super(TableRowkeyPair.class); + } + + @Override + public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { + try { + // Compare table names + int strL1 = readInt(b1, s1); + int strL2 = readInt(b2, s2); + int cmp = compareBytes(b1, s1 + Bytes.SIZEOF_INT, strL1, b2, s2 + Bytes.SIZEOF_INT, strL2); + if (cmp != 0) { + return cmp; + } + // Compare row keys + int strL3 = readInt(b1, s1 + Bytes.SIZEOF_INT + strL1); + int strL4 = readInt(b2, s2 + Bytes.SIZEOF_INT + strL2); + int i = compareBytes(b1, s1 + Bytes.SIZEOF_INT*2 + strL1, strL3, b2, s2 + + Bytes.SIZEOF_INT*2 + strL2, strL4); + return i; + } catch(Exception ex) { + throw new IllegalArgumentException(ex); + } + } + } + + static { + WritableComparator.define(TableRowkeyPair.class, new Comparator()); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a82a0ff6/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java index 1fee4bb..2a29c00 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java @@ -23,7 +23,6 @@ import java.io.IOException; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.BytesWritable; import org.junit.Assert; import org.junit.Test; @@ -38,8 +37,9 @@ public class TestTableRowkeyPair { testsRowsKeys("first", "aa", "first", "ab", -1); testsRowsKeys("second", "aa", "first", "aa", 1); testsRowsKeys("first", "aa", "first", "aaa", -1); + testsRowsKeys("first","bb", "first", "aaaa", 1); } - + private void testsRowsKeys(String aTable, String akey, String bTable, String bkey, int expectedSignum) throws IOException { final ImmutableBytesWritable arowkey = new ImmutableBytesWritable(Bytes.toBytes(akey)); @@ -48,7 +48,7 @@ public class TestTableRowkeyPair { ImmutableBytesWritable browkey = new ImmutableBytesWritable(Bytes.toBytes(bkey)); TableRowkeyPair pair2 = new TableRowkeyPair(bTable, browkey); - BytesWritable.Comparator comparator = new BytesWritable.Comparator(); + TableRowkeyPair.Comparator comparator = new TableRowkeyPair.Comparator(); try( ByteArrayOutputStream baosA = new ByteArrayOutputStream(); ByteArrayOutputStream baosB = new ByteArrayOutputStream()) {
