Merge branch '1.7' into 1.8 Conflicts: core/src/test/java/org/apache/accumulo/core/data/RangeTest.java core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java core/src/test/java/org/apache/accumulo/core/iterators/system/MultiIteratorTest.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cd70d781 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cd70d781 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cd70d781 Branch: refs/heads/master Commit: cd70d7815afad9ae7d0f78c5fa916833b274690b Parents: 43b740b a4a4539 Author: Mike Miller <mmil...@apache.org> Authored: Tue Nov 22 14:58:58 2016 -0500 Committer: Mike Miller <mmil...@apache.org> Committed: Tue Nov 22 14:58:58 2016 -0500 ---------------------------------------------------------------------- .../apache/accumulo/core/data/RangeTest.java | 428 ++++++++++--------- .../core/file/rfile/BlockIndexTest.java | 22 +- .../core/file/rfile/CreateCompatTestFile.java | 20 +- .../core/file/rfile/MultiThreadedRFileTest.java | 8 +- .../core/file/rfile/RFileMetricsTest.java | 78 ++-- .../accumulo/core/file/rfile/RFileTest.java | 416 +++++++++--------- .../core/file/rfile/RelativeKeyTest.java | 14 +- .../core/iterators/AggregatingIteratorTest.java | 152 +++---- .../core/iterators/FirstEntryInRowTest.java | 46 +- .../core/iterators/IteratorUtilTest.java | 38 +- .../ColumnFamilySkippingIteratorTest.java | 44 +- .../core/iterators/system/ColumnFilterTest.java | 30 +- .../iterators/system/DeletingIteratorTest.java | 62 +-- .../iterators/system/MultiIteratorTest.java | 125 +++--- .../system/SourceSwitchingIteratorTest.java | 45 +- .../iterators/user/BigDecimalCombinerTest.java | 20 +- .../iterators/user/ColumnSliceFilterTest.java | 18 +- .../core/iterators/user/CombinerTest.java | 220 +++++----- .../iterators/user/IndexedDocIteratorTest.java | 4 +- .../core/iterators/user/RegExFilterTest.java | 12 +- .../iterators/user/RowDeletingIteratorTest.java | 80 ++-- .../tserver/CheckTabletMetadataTest.java | 14 +- .../accumulo/tserver/InMemoryMapTest.java | 178 ++++---- .../accumulo/test/NativeMapPerformanceTest.java | 12 +- .../accumulo/test/functional/NativeMapIT.java | 79 ++-- 25 files changed, 1086 insertions(+), 1079 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/data/RangeTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/data/RangeTest.java index c4837fe,115aa7a..fe7f1f8 --- a/core/src/test/java/org/apache/accumulo/core/data/RangeTest.java +++ b/core/src/test/java/org/apache/accumulo/core/data/RangeTest.java @@@ -191,30 -191,31 +191,31 @@@ public class RangeTest extends TestCas public void testMergeOverlapping22() { - Range ke1 = new KeyExtent(new Text("tab1"), new Text("Bank"), null).toMetadataRange(); - Range ke2 = new KeyExtent(new Text("tab1"), new Text("Fails"), new Text("Bank")).toMetadataRange(); - Range ke3 = new KeyExtent(new Text("tab1"), new Text("Sam"), new Text("Fails")).toMetadataRange(); - Range ke4 = new KeyExtent(new Text("tab1"), new Text("bails"), new Text("Sam")).toMetadataRange(); - Range ke5 = new KeyExtent(new Text("tab1"), null, new Text("bails")).toMetadataRange(); + Range ke1 = new KeyExtent("tab1", new Text("Bank"), null).toMetadataRange(); + Range ke2 = new KeyExtent("tab1", new Text("Fails"), new Text("Bank")).toMetadataRange(); + Range ke3 = new KeyExtent("tab1", new Text("Sam"), new Text("Fails")).toMetadataRange(); + Range ke4 = new KeyExtent("tab1", new Text("bails"), new Text("Sam")).toMetadataRange(); + Range ke5 = new KeyExtent("tab1", null, new Text("bails")).toMetadataRange(); - List<Range> rl = nrl(ke1, ke2, ke3, ke4, ke5); - List<Range> expected = nrl(new KeyExtent("tab1", null, null).toMetadataRange()); + List<Range> rl = newRangeList(ke1, ke2, ke3, ke4, ke5); - List<Range> expected = newRangeList(new KeyExtent(new Text("tab1"), null, null).toMetadataRange()); ++ List<Range> expected = newRangeList(new KeyExtent("tab1", null, null).toMetadataRange()); check(Range.mergeOverlapping(rl), expected); - rl = nrl(ke1, ke2, ke4, ke5); - expected = nrl(new KeyExtent("tab1", new Text("Fails"), null).toMetadataRange(), new KeyExtent("tab1", null, new Text("Sam")).toMetadataRange()); + rl = newRangeList(ke1, ke2, ke4, ke5); - expected = newRangeList(new KeyExtent(new Text("tab1"), new Text("Fails"), null).toMetadataRange(), - new KeyExtent(new Text("tab1"), null, new Text("Sam")).toMetadataRange()); ++ expected = newRangeList(new KeyExtent("tab1", new Text("Fails"), null).toMetadataRange(), new KeyExtent("tab1", null, new Text("Sam")).toMetadataRange()); check(Range.mergeOverlapping(rl), expected); - rl = nrl(ke2, ke3, ke4, ke5); - expected = nrl(new KeyExtent("tab1", null, new Text("Bank")).toMetadataRange()); + rl = newRangeList(ke2, ke3, ke4, ke5); - expected = newRangeList(new KeyExtent(new Text("tab1"), null, new Text("Bank")).toMetadataRange()); ++ expected = newRangeList(new KeyExtent("tab1", null, new Text("Bank")).toMetadataRange()); check(Range.mergeOverlapping(rl), expected); - rl = nrl(ke1, ke2, ke3, ke4); - expected = nrl(new KeyExtent("tab1", new Text("bails"), null).toMetadataRange()); + rl = newRangeList(ke1, ke2, ke3, ke4); - expected = newRangeList(new KeyExtent(new Text("tab1"), new Text("bails"), null).toMetadataRange()); ++ expected = newRangeList(new KeyExtent("tab1", new Text("bails"), null).toMetadataRange()); check(Range.mergeOverlapping(rl), expected); - rl = nrl(ke2, ke3, ke4); - expected = nrl(new KeyExtent("tab1", new Text("bails"), new Text("Bank")).toMetadataRange()); + rl = newRangeList(ke2, ke3, ke4); - expected = newRangeList(new KeyExtent(new Text("tab1"), new Text("bails"), new Text("Bank")).toMetadataRange()); ++ expected = newRangeList(new KeyExtent("tab1", new Text("bails"), new Text("Bank")).toMetadataRange()); ++ check(Range.mergeOverlapping(rl), expected); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java index e7c8b46,eec4e71..34d01ad --- a/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java +++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java @@@ -56,20 -56,20 +56,20 @@@ public class CreateCompatTestFile public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); - CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(fs, new Path(args[0]), "gz", conf, AccumuloConfiguration.getDefaultConfiguration()); + CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(fs, new Path(args[0]), "gz", null, conf, AccumuloConfiguration.getDefaultConfiguration()); RFile.Writer writer = new RFile.Writer(_cbw, 1000); - writer.startNewLocalityGroup("lg1", ncfs(nf("cf_", 1), nf("cf_", 2))); + writer.startNewLocalityGroup("lg1", newColFamSequence(formatStr("cf_", 1), formatStr("cf_", 2))); for (int i = 0; i < 1000; i++) { - writer.append(nk(nf("r_", i), nf("cf_", 1), nf("cq_", 0), "", 1000 - i), nv(i + "")); - writer.append(nk(nf("r_", i), nf("cf_", 2), nf("cq_", 0), "", 1000 - i), nv(i + "")); + writer.append(newKey(formatStr("r_", i), formatStr("cf_", 1), formatStr("cq_", 0), "", 1000 - i), newValue(i + "")); + writer.append(newKey(formatStr("r_", i), formatStr("cf_", 2), formatStr("cq_", 0), "", 1000 - i), newValue(i + "")); } - writer.startNewLocalityGroup("lg2", ncfs(nf("cf_", 3))); + writer.startNewLocalityGroup("lg2", newColFamSequence(formatStr("cf_", 3))); for (int i = 0; i < 1000; i++) { - writer.append(nk(nf("r_", i), nf("cf_", 3), nf("cq_", 0), "", 1000 - i), nv(i + "")); + writer.append(newKey(formatStr("r_", i), formatStr("cf_", 3), formatStr("cq_", 0), "", 1000 - i), newValue(i + "")); } writer.startDefaultLocalityGroup(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileMetricsTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java index 3833564,96f97e7..aff1a8f --- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java @@@ -378,18 -338,18 +378,18 @@@ public class RFileTest for (int cv = 'A'; cv < 'A' + 4; cv++) { String cvS = "" + (char) cv; for (int ts = 4; ts > 0; ts--) { - Key k = nk(rowS, cfS, cqS, cvS, ts); + Key k = newKey(rowS, cfS, cqS, cvS, ts); // check below ensures when all key sizes are same more than one index block is created - assertEquals(27, k.getSize()); + Assert.assertEquals(27, k.getSize()); k.setDeleted(true); - Value v = nv("" + val); + Value v = newValue("" + val); trf.writer.append(k, v); expectedKeys.add(k); expectedValues.add(v); - k = nk(rowS, cfS, cqS, cvS, ts); + k = newKey(rowS, cfS, cqS, cvS, ts); - assertEquals(27, k.getSize()); + Assert.assertEquals(27, k.getSize()); - v = nv("" + val); + v = newValue("" + val); trf.writer.append(k, v); expectedKeys.add(k); expectedValues.add(v); @@@ -531,39 -491,39 +531,39 @@@ trf.openWriter(); - trf.writer.append(nk("r1", "cf1", "cq1", "L1", 55), nv("foo1")); + trf.writer.append(newKey("r1", "cf1", "cq1", "L1", 55), newValue("foo1")); try { - trf.writer.append(nk("r0", "cf1", "cq1", "L1", 55), nv("foo1")); + trf.writer.append(newKey("r0", "cf1", "cq1", "L1", 55), newValue("foo1")); assertFalse(true); - } catch (IllegalStateException ioe) { + } catch (IllegalArgumentException ioe) { } try { - trf.writer.append(nk("r1", "cf0", "cq1", "L1", 55), nv("foo1")); + trf.writer.append(newKey("r1", "cf0", "cq1", "L1", 55), newValue("foo1")); assertFalse(true); - } catch (IllegalStateException ioe) { + } catch (IllegalArgumentException ioe) { } try { - trf.writer.append(nk("r1", "cf1", "cq0", "L1", 55), nv("foo1")); + trf.writer.append(newKey("r1", "cf1", "cq0", "L1", 55), newValue("foo1")); assertFalse(true); - } catch (IllegalStateException ioe) { + } catch (IllegalArgumentException ioe) { } try { - trf.writer.append(nk("r1", "cf1", "cq1", "L0", 55), nv("foo1")); + trf.writer.append(newKey("r1", "cf1", "cq1", "L0", 55), newValue("foo1")); assertFalse(true); - } catch (IllegalStateException ioe) { + } catch (IllegalArgumentException ioe) { } try { - trf.writer.append(nk("r1", "cf1", "cq1", "L1", 56), nv("foo1")); + trf.writer.append(newKey("r1", "cf1", "cq1", "L1", 56), newValue("foo1")); assertFalse(true); - } catch (IllegalStateException ioe) { + } catch (IllegalArgumentException ioe) { } } @@@ -646,38 -606,44 +646,38 @@@ trf.openReader(); // test that has top returns false when end of range reached - trf.iter.seek(new Range(nk(nf("r_", 3), "cf1", "cq1", "L1", 55), true, nk(nf("r_", 4), "cf1", "cq1", "L1", 55), false), EMPTY_COL_FAMS, false); - trf.iter.seek(new Range(newKey(formatString("r_", 3), "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 4), "cf1", "cq1", "L1", 55), false), - EMPTY_COL_FAMS, false); ++ trf.iter.seek(new Range(newKey(formatString("r_", 3), "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 4), "cf1", "cq1", "L1", 55), false), EMPTY_COL_FAMS, false); assertTrue(trf.iter.hasTop()); - assertTrue(trf.iter.getTopKey().equals(nk(nf("r_", 3), "cf1", "cq1", "L1", 55))); - assertEquals(nv("foo" + 3), trf.iter.getTopValue()); + assertTrue(trf.iter.getTopKey().equals(newKey(formatString("r_", 3), "cf1", "cq1", "L1", 55))); + assertEquals(newValue("foo" + 3), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); // test seeking to a range that is between two keys, should not return anything - trf.iter.seek(new Range(nk(nf("r_", 4) + "a", "cf1", "cq1", "L1", 55), true, nk(nf("r_", 4) + "b", "cf1", "cq1", "L1", 55), true), EMPTY_COL_FAMS, false); - trf.iter.seek(new Range(newKey(formatString("r_", 4) + "a", "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 4) + "b", "cf1", "cq1", "L1", 55), - true), EMPTY_COL_FAMS, false); ++ trf.iter.seek(new Range(newKey(formatString("r_", 4) + "a", "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 4) + "b", "cf1", "cq1", "L1", 55), true), EMPTY_COL_FAMS, false); assertFalse(trf.iter.hasTop()); // test seeking to another range after the previously seeked range, that is between the same two keys in the file // as the previously seeked range.... this test an optimization on RFile - trf.iter.seek(new Range(nk(nf("r_", 4) + "c", "cf1", "cq1", "L1", 55), true, nk(nf("r_", 4) + "d", "cf1", "cq1", "L1", 55), true), EMPTY_COL_FAMS, false); - trf.iter.seek(new Range(newKey(formatString("r_", 4) + "c", "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 4) + "d", "cf1", "cq1", "L1", 55), - true), EMPTY_COL_FAMS, false); ++ trf.iter.seek(new Range(newKey(formatString("r_", 4) + "c", "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 4) + "d", "cf1", "cq1", "L1", 55), true), EMPTY_COL_FAMS, false); assertFalse(trf.iter.hasTop()); - trf.iter.seek(new Range(nk(nf("r_", 4) + "e", "cf1", "cq1", "L1", 55), true, nk(nf("r_", 4) + "f", "cf1", "cq1", "L1", 55), true), EMPTY_COL_FAMS, false); - trf.iter.seek(new Range(newKey(formatString("r_", 4) + "e", "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 4) + "f", "cf1", "cq1", "L1", 55), - true), EMPTY_COL_FAMS, false); ++ trf.iter.seek(new Range(newKey(formatString("r_", 4) + "e", "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 4) + "f", "cf1", "cq1", "L1", 55), true), EMPTY_COL_FAMS, false); assertFalse(trf.iter.hasTop()); // now ensure we can seek somewhere, that triggering the optimization does not cause any problems - trf.iter.seek(new Range(nk(nf("r_", 5), "cf1", "cq1", "L1", 55), true, nk(nf("r_", 6), "cf1", "cq1", "L1", 55), false), EMPTY_COL_FAMS, false); - trf.iter.seek(new Range(newKey(formatString("r_", 5), "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 6), "cf1", "cq1", "L1", 55), false), - EMPTY_COL_FAMS, false); ++ trf.iter.seek(new Range(newKey(formatString("r_", 5), "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 6), "cf1", "cq1", "L1", 55), false), EMPTY_COL_FAMS, false); assertTrue(trf.iter.hasTop()); - assertTrue(trf.iter.getTopKey().equals(nk(nf("r_", 5), "cf1", "cq1", "L1", 55))); - assertEquals(nv("foo" + 5), trf.iter.getTopValue()); + assertTrue(trf.iter.getTopKey().equals(newKey(formatString("r_", 5), "cf1", "cq1", "L1", 55))); + assertEquals(newValue("foo" + 5), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); // test seeking to range that is before the beginning of the file - trf.iter.seek(new Range(nk(nf("r_", 0), "cf1", "cq1", "L1", 55), true, nk(nf("r_", 2), "cf1", "cq1", "L1", 55), false), EMPTY_COL_FAMS, false); - trf.iter.seek(new Range(newKey(formatString("r_", 0), "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 2), "cf1", "cq1", "L1", 55), false), - EMPTY_COL_FAMS, false); ++ trf.iter.seek(new Range(newKey(formatString("r_", 0), "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 2), "cf1", "cq1", "L1", 55), false), EMPTY_COL_FAMS, false); assertFalse(trf.iter.hasTop()); - assertEquals(nk(nf("r_", 49), "cf1", "cq1", "L1", 55), trf.reader.getLastKey()); + assertEquals(newKey(formatString("r_", 49), "cf1", "cq1", "L1", 55), trf.reader.getLastKey()); trf.reader.close(); } @@@ -805,23 -771,23 +805,23 @@@ trf.openWriter(false); - trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("cf1", "cf2")); + trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2")); - trf.writer.append(nk("0000", "cf1", "doe,john", "", 4), nv("1123 West Left st")); - trf.writer.append(nk("0002", "cf2", "doe,jane", "", 5), nv("1124 East Right st")); + trf.writer.append(newKey("0000", "cf1", "doe,john", "", 4), newValue("1123 West Left st")); + trf.writer.append(newKey("0002", "cf2", "doe,jane", "", 5), newValue("1124 East Right st")); - trf.writer.startNewLocalityGroup("lg2", newColFamByteSequence("cf3", "cf4")); + trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4")); - trf.writer.append(nk("0001", "cf3", "buck,john", "", 4), nv("90 Slum st")); - trf.writer.append(nk("0003", "cf4", "buck,jane", "", 5), nv("09 Slum st")); + trf.writer.append(newKey("0001", "cf3", "buck,john", "", 4), newValue("90 Slum st")); + trf.writer.append(newKey("0003", "cf4", "buck,jane", "", 5), newValue("09 Slum st")); trf.writer.close(); trf.openReader(); // scan first loc group - Range r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); + Range r = new Range(newKey("0000", "cf1", "doe,john", "", 4), true, newKey("0003", "cf4", "buck,jane", "", 5), true); - trf.iter.seek(r, newColFamByteSequence("cf1", "cf2"), true); + trf.iter.seek(r, ncfs("cf1", "cf2"), true); assertEquals(1, trf.reader.getNumLocalityGroupsSeeked()); assertTrue(trf.iter.hasTop()); @@@ -835,8 -801,8 +835,8 @@@ assertFalse(trf.iter.hasTop()); // scan second loc group - r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); + r = new Range(newKey("0000", "cf1", "doe,john", "", 4), true, newKey("0003", "cf4", "buck,jane", "", 5), true); - trf.iter.seek(r, newColFamByteSequence("cf3", "cf4"), true); + trf.iter.seek(r, ncfs("cf3", "cf4"), true); assertEquals(1, trf.reader.getNumLocalityGroupsSeeked()); assertTrue(trf.iter.hasTop()); @@@ -873,14 -839,14 +873,14 @@@ assertFalse(trf.iter.hasTop()); // scan no loc groups - r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); + r = new Range(newKey("0000", "cf1", "doe,john", "", 4), true, newKey("0003", "cf4", "buck,jane", "", 5), true); - trf.iter.seek(r, newColFamByteSequence("saint", "dogooder"), true); + trf.iter.seek(r, ncfs("saint", "dogooder"), true); assertEquals(0, trf.reader.getNumLocalityGroupsSeeked()); assertFalse(trf.iter.hasTop()); // scan a subset of second locality group - r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); + r = new Range(newKey("0000", "cf1", "doe,john", "", 4), true, newKey("0003", "cf4", "buck,jane", "", 5), true); - trf.iter.seek(r, newColFamByteSequence("cf4"), true); + trf.iter.seek(r, ncfs("cf4"), true); assertEquals(1, trf.reader.getNumLocalityGroupsSeeked()); assertTrue(trf.iter.hasTop()); @@@ -890,8 -856,8 +890,8 @@@ assertFalse(trf.iter.hasTop()); // scan a subset of second locality group - r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); + r = new Range(newKey("0000", "cf1", "doe,john", "", 4), true, newKey("0003", "cf4", "buck,jane", "", 5), true); - trf.iter.seek(r, newColFamByteSequence("cf3"), true); + trf.iter.seek(r, ncfs("cf3"), true); assertEquals(1, trf.reader.getNumLocalityGroupsSeeked()); assertTrue(trf.iter.hasTop()); @@@ -901,8 -867,8 +901,8 @@@ assertFalse(trf.iter.hasTop()); // scan subset of first loc group - r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); + r = new Range(newKey("0000", "cf1", "doe,john", "", 4), true, newKey("0003", "cf4", "buck,jane", "", 5), true); - trf.iter.seek(r, newColFamByteSequence("cf1"), true); + trf.iter.seek(r, ncfs("cf1"), true); assertEquals(1, trf.reader.getNumLocalityGroupsSeeked()); assertTrue(trf.iter.hasTop()); @@@ -912,8 -878,8 +912,8 @@@ assertFalse(trf.iter.hasTop()); // scan subset of first loc group - r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); + r = new Range(newKey("0000", "cf1", "doe,john", "", 4), true, newKey("0003", "cf4", "buck,jane", "", 5), true); - trf.iter.seek(r, newColFamByteSequence("cf2"), true); + trf.iter.seek(r, ncfs("cf2"), true); assertEquals(1, trf.reader.getNumLocalityGroupsSeeked()); assertTrue(trf.iter.hasTop()); @@@ -923,8 -889,8 +923,8 @@@ assertFalse(trf.iter.hasTop()); // scan subset of all loc groups - r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); + r = new Range(newKey("0000", "cf1", "doe,john", "", 4), true, newKey("0003", "cf4", "buck,jane", "", 5), true); - trf.iter.seek(r, newColFamByteSequence("cf1", "cf4"), true); + trf.iter.seek(r, ncfs("cf1", "cf4"), true); assertEquals(2, trf.reader.getNumLocalityGroupsSeeked()); assertTrue(trf.iter.hasTop()); @@@ -963,10 -929,10 +963,10 @@@ trf = new TestRFile(conf); trf.openWriter(false); - trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("cf1", "cf2")); + trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2")); - trf.writer.append(nk("0000", "cf1", "doe,john", "", 4), nv("1123 West Left st")); - trf.writer.append(nk("0002", "cf2", "doe,jane", "", 5), nv("1124 East Right st")); + trf.writer.append(newKey("0000", "cf1", "doe,john", "", 4), newValue("1123 West Left st")); + trf.writer.append(newKey("0002", "cf2", "doe,jane", "", 5), newValue("1124 East Right st")); - trf.writer.startNewLocalityGroup("lg2", newColFamByteSequence("cf3", "cf4")); + trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4")); trf.writer.startDefaultLocalityGroup(); trf.writer.close(); @@@ -988,10 -954,10 +988,10 @@@ trf = new TestRFile(conf); trf.openWriter(false); - trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("cf1", "cf2")); - trf.writer.startNewLocalityGroup("lg2", newColFamByteSequence("cf3", "cf4")); + trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2")); + trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4")); - trf.writer.append(nk("0001", "cf3", "buck,john", "", 4), nv("90 Slum st")); - trf.writer.append(nk("0003", "cf4", "buck,jane", "", 5), nv("09 Slum st")); + trf.writer.append(newKey("0001", "cf3", "buck,john", "", 4), newValue("90 Slum st")); + trf.writer.append(newKey("0003", "cf4", "buck,jane", "", 5), newValue("09 Slum st")); trf.writer.startDefaultLocalityGroup(); trf.writer.close(); @@@ -1013,11 -979,11 +1013,11 @@@ trf = new TestRFile(conf); trf.openWriter(false); - trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("cf1", "cf2")); - trf.writer.startNewLocalityGroup("lg2", newColFamByteSequence("cf3", "cf4")); + trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2")); + trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4")); trf.writer.startDefaultLocalityGroup(); - trf.writer.append(nk("0007", "good citizen", "q,john", "", 4), nv("70 Apple st")); - trf.writer.append(nk("0008", "model citizen", "q,jane", "", 5), nv("81 Plum st")); + trf.writer.append(newKey("0007", "good citizen", "q,john", "", 4), newValue("70 Apple st")); + trf.writer.append(newKey("0008", "model citizen", "q,jane", "", 5), newValue("81 Plum st")); trf.writer.close(); trf.openReader(); @@@ -1038,13 -1004,13 +1038,13 @@@ trf = new TestRFile(conf); trf.openWriter(false); - trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("cf1", "cf2")); + trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2")); - trf.writer.append(nk("0000", "cf1", "doe,john", "", 4), nv("1123 West Left st")); - trf.writer.append(nk("0002", "cf2", "doe,jane", "", 5), nv("1124 East Right st")); + trf.writer.append(newKey("0000", "cf1", "doe,john", "", 4), newValue("1123 West Left st")); + trf.writer.append(newKey("0002", "cf2", "doe,jane", "", 5), newValue("1124 East Right st")); - trf.writer.startNewLocalityGroup("lg2", newColFamByteSequence("cf3", "cf4")); + trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4")); trf.writer.startDefaultLocalityGroup(); - trf.writer.append(nk("0007", "good citizen", "q,john", "", 4), nv("70 Apple st")); - trf.writer.append(nk("0008", "model citizen", "q,jane", "", 5), nv("81 Plum st")); + trf.writer.append(newKey("0007", "good citizen", "q,john", "", 4), newValue("70 Apple st")); + trf.writer.append(newKey("0008", "model citizen", "q,jane", "", 5), newValue("81 Plum st")); trf.writer.close(); trf.openReader(); @@@ -1077,17 -1043,17 +1077,17 @@@ TestRFile trf = new TestRFile(conf); trf.openWriter(false); - trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("3mod10")); + trf.writer.startNewLocalityGroup("lg1", ncfs("3mod10")); for (int i = 3; i < 1024; i += 10) { - trf.writer.append(nk(nf("i", i), "3mod10", "", "", i + 2), nv("" + i)); + trf.writer.append(newKey(formatString("i", i), "3mod10", "", "", i + 2), newValue("" + i)); } - trf.writer.startNewLocalityGroup("lg2", newColFamByteSequence("5mod10", "7mod10")); + trf.writer.startNewLocalityGroup("lg2", ncfs("5mod10", "7mod10")); for (int i = 5; i < 1024;) { - trf.writer.append(nk(nf("i", i), "5mod10", "", "", i + 2), nv("" + i)); + trf.writer.append(newKey(formatString("i", i), "5mod10", "", "", i + 2), newValue("" + i)); i += 2; - trf.writer.append(nk(nf("i", i), "7mod10", "", "", i + 2), nv("" + i)); + trf.writer.append(newKey(formatString("i", i), "7mod10", "", "", i + 2), newValue("" + i)); i += 8; } @@@ -1183,12 -1149,12 +1183,12 @@@ trf.openWriter(false); - trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("a", "b")); + trf.writer.startNewLocalityGroup("lg1", ncfs("a", "b")); - trf.writer.append(nk("0007", "a", "cq1", "", 4), nv("1")); + trf.writer.append(newKey("0007", "a", "cq1", "", 4), newValue("1")); try { - trf.writer.append(nk("0009", "c", "cq1", "", 4), nv("1")); + trf.writer.append(newKey("0009", "c", "cq1", "", 4), newValue("1")); assertFalse(true); } catch (IllegalArgumentException ioe) { @@@ -1216,9 -1182,9 +1216,9 @@@ trf.openWriter(false); - trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("a", "b")); + trf.writer.startNewLocalityGroup("lg1", ncfs("a", "b")); - trf.writer.append(nk("0007", "a", "cq1", "", 4), nv("1")); + trf.writer.append(newKey("0007", "a", "cq1", "", 4), newValue("1")); trf.writer.startDefaultLocalityGroup(); @@@ -1280,11 -1247,11 +1280,11 @@@ trf.openWriter(false); - trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("a", "b")); + trf.writer.startNewLocalityGroup("lg1", ncfs("a", "b")); - trf.writer.append(nk("0007", "a", "cq1", "", 4), nv("1")); + trf.writer.append(newKey("0007", "a", "cq1", "", 4), newValue("1")); try { - trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("b", "c")); + trf.writer.startNewLocalityGroup("lg1", ncfs("b", "c")); assertFalse(true); } catch (IllegalArgumentException ioe) { @@@ -1492,10 -1459,10 +1492,10 @@@ trf.openWriter(false); trf.openWriter(false); - trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("cf1", "cf2")); + trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2")); - trf.writer.append(nk("0000", "cf1", "doe,john", "", 4), nv("1123 West Left st")); - trf.writer.append(nk("0002", "cf2", "doe,jane", "", 5), nv("1124 East Right st")); + trf.writer.append(newKey("0000", "cf1", "doe,john", "", 4), newValue("1123 West Left st")); + trf.writer.append(newKey("0002", "cf2", "doe,jane", "", 5), newValue("1124 East Right st")); - trf.writer.startNewLocalityGroup("lg2", newColFamByteSequence("cf3", "cf4")); + trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4")); DataOutputStream dos = trf.writer.createMetaStore("count"); @@@ -1627,9 -1593,9 +1627,9 @@@ for (int start : new int[] {0, 10, 100, 998}) { for (int cf = 1; cf <= 4; cf++) { if (start == 0) - iter.seek(new Range(), ncfs(nf("cf_", cf)), true); - iter.seek(new Range(), newColFamByteSequence(formatString("cf_", cf)), true); ++ iter.seek(new Range(), ncfs(formatString("cf_", cf)), true); else - iter.seek(new Range(nf("r_", start), null), ncfs(nf("cf_", cf)), true); - iter.seek(new Range(formatString("r_", start), null), newColFamByteSequence(formatString("cf_", cf)), true); ++ iter.seek(new Range(formatString("r_", start), null), ncfs(formatString("cf_", cf)), true); for (int i = start; i < 1000; i++) { assertTrue(iter.hasTop()); @@@ -1642,9 -1608,9 +1642,9 @@@ } if (start == 0) - iter.seek(new Range(), newColFamByteSequence(), false); + iter.seek(new Range(), ncfs(), false); else - iter.seek(new Range(nf("r_", start), null), ncfs(), false); - iter.seek(new Range(formatString("r_", start), null), newColFamByteSequence(), false); ++ iter.seek(new Range(formatString("r_", start), null), ncfs(), false); for (int i = start; i < 1000; i++) { for (int cf = 1; cf <= 4; cf++) { @@@ -1808,294 -1774,6 +1808,294 @@@ conf = null; } - private Key nk(int r, int c) { ++ private Key newKey(int r, int c) { + String row = String.format("r%06d", r); + switch (c) { + case 0: + return new Key(row, "user", "addr"); + case 1: + return new Key(row, "user", "name"); + default: + throw new IllegalArgumentException(); + } + } + - private Value nv(int r, int c) { ++ private Value newValue(int r, int c) { + switch (c) { + case 0: + return new Value(("123" + r + " west st").getBytes()); + case 1: + return new Value(("bob" + r).getBytes()); + default: + throw new IllegalArgumentException(); + } + } + + private static void hash(Hasher hasher, Key key, Value val) { + hasher.putBytes(key.getRowData().toArray()); + hasher.putBytes(key.getColumnFamilyData().toArray()); + hasher.putBytes(key.getColumnQualifierData().toArray()); + hasher.putBytes(key.getColumnVisibilityData().toArray()); + hasher.putLong(key.getTimestamp()); + hasher.putBoolean(key.isDeleted()); + hasher.putBytes(val.get()); + } + + private static void add(TestRFile trf, Key key, Value val, Hasher dataHasher, List<Entry<Key,Value>> sample, Sampler sampler) throws IOException { + if (sampler.accept(key)) { + sample.add(new AbstractMap.SimpleImmutableEntry<>(key, val)); + } + + hash(dataHasher, key, val); + + trf.writer.append(key, val); + } + + private List<Entry<Key,Value>> toList(SortedKeyValueIterator<Key,Value> sample) throws IOException { + ArrayList<Entry<Key,Value>> ret = new ArrayList<>(); + + while (sample.hasTop()) { + ret.add(new AbstractMap.SimpleImmutableEntry<>(new Key(sample.getTopKey()), new Value(sample.getTopValue()))); + sample.next(); + } + + return ret; + } + + private void checkSample(SortedKeyValueIterator<Key,Value> sample, List<Entry<Key,Value>> sampleData) throws IOException { + checkSample(sample, sampleData, EMPTY_COL_FAMS, false); + } + + private void checkSample(SortedKeyValueIterator<Key,Value> sample, List<Entry<Key,Value>> sampleData, Collection<ByteSequence> columnFamilies, + boolean inclusive) throws IOException { + + sample.seek(new Range(), columnFamilies, inclusive); + Assert.assertEquals(sampleData, toList(sample)); + + Random rand = new Random(); + long seed = rand.nextLong(); + rand = new Random(seed); + + // randomly seek sample iterator and verify + for (int i = 0; i < 33; i++) { + Key startKey = null; + boolean startInclusive = false; + int startIndex = 0; + + Key endKey = null; + boolean endInclusive = false; + int endIndex = sampleData.size(); + + if (rand.nextBoolean()) { + startIndex = rand.nextInt(sampleData.size()); + startKey = sampleData.get(startIndex).getKey(); + startInclusive = rand.nextBoolean(); + if (!startInclusive) { + startIndex++; + } + } + + if (startIndex < endIndex && rand.nextBoolean()) { + endIndex -= rand.nextInt(endIndex - startIndex); + endKey = sampleData.get(endIndex - 1).getKey(); + endInclusive = rand.nextBoolean(); + if (!endInclusive) { + endIndex--; + } + } else if (startIndex == endIndex) { + endInclusive = rand.nextBoolean(); + } + + sample.seek(new Range(startKey, startInclusive, endKey, endInclusive), columnFamilies, inclusive); + Assert.assertEquals("seed: " + seed, sampleData.subList(startIndex, endIndex), toList(sample)); + } + } + + @Test + public void testSample() throws IOException { + + int num = 10000; + + for (int sampleBufferSize : new int[] {1 << 10, 1 << 20}) { + // force sample buffer to flush for smaller data + RFile.setSampleBufferSize(sampleBufferSize); + + for (int modulus : new int[] {19, 103, 1019}) { + Hasher dataHasher = Hashing.md5().newHasher(); + List<Entry<Key,Value>> sampleData = new ArrayList<>(); + + ConfigurationCopy sampleConf = new ConfigurationCopy(conf == null ? AccumuloConfiguration.getDefaultConfiguration() : conf); + sampleConf.set(Property.TABLE_SAMPLER, RowSampler.class.getName()); + sampleConf.set(Property.TABLE_SAMPLER_OPTS + "hasher", "murmur3_32"); + sampleConf.set(Property.TABLE_SAMPLER_OPTS + "modulus", modulus + ""); + + Sampler sampler = SamplerFactory.newSampler(SamplerConfigurationImpl.newSamplerConfig(sampleConf), sampleConf); + + TestRFile trf = new TestRFile(sampleConf); + + trf.openWriter(); + + for (int i = 0; i < num; i++) { - add(trf, nk(i, 0), nv(i, 0), dataHasher, sampleData, sampler); - add(trf, nk(i, 1), nv(i, 1), dataHasher, sampleData, sampler); ++ add(trf, newKey(i, 0), newValue(i, 0), dataHasher, sampleData, sampler); ++ add(trf, newKey(i, 1), newValue(i, 1), dataHasher, sampleData, sampler); + } + + HashCode expectedDataHash = dataHasher.hash(); + + trf.closeWriter(); + + trf.openReader(); + + FileSKVIterator sample = trf.reader.getSample(SamplerConfigurationImpl.newSamplerConfig(sampleConf)); + + checkSample(sample, sampleData); + + Assert.assertEquals(expectedDataHash, hash(trf.reader)); + + SampleIE ie = new SampleIE(SamplerConfigurationImpl.newSamplerConfig(sampleConf).toSamplerConfiguration()); + + for (int i = 0; i < 3; i++) { + // test opening and closing deep copies a few times. + trf.reader.closeDeepCopies(); + + sample = trf.reader.getSample(SamplerConfigurationImpl.newSamplerConfig(sampleConf)); + SortedKeyValueIterator<Key,Value> sampleDC1 = sample.deepCopy(ie); + SortedKeyValueIterator<Key,Value> sampleDC2 = sample.deepCopy(ie); + SortedKeyValueIterator<Key,Value> sampleDC3 = trf.reader.deepCopy(ie); + SortedKeyValueIterator<Key,Value> allDC1 = sampleDC1.deepCopy(new SampleIE(null)); + SortedKeyValueIterator<Key,Value> allDC2 = sample.deepCopy(new SampleIE(null)); + + Assert.assertEquals(expectedDataHash, hash(allDC1)); + Assert.assertEquals(expectedDataHash, hash(allDC2)); + + checkSample(sample, sampleData); + checkSample(sampleDC1, sampleData); + checkSample(sampleDC2, sampleData); + checkSample(sampleDC3, sampleData); + } + + trf.reader.closeDeepCopies(); + + trf.closeReader(); + } + } + } + + private HashCode hash(SortedKeyValueIterator<Key,Value> iter) throws IOException { + Hasher dataHasher = Hashing.md5().newHasher(); + iter.seek(new Range(), EMPTY_COL_FAMS, false); + while (iter.hasTop()) { + hash(dataHasher, iter.getTopKey(), iter.getTopValue()); + iter.next(); + } + + return dataHasher.hash(); + } + + @Test + public void testSampleLG() throws IOException { + + int num = 5000; + + for (int sampleBufferSize : new int[] {1 << 10, 1 << 20}) { + // force sample buffer to flush for smaller data + RFile.setSampleBufferSize(sampleBufferSize); + + for (int modulus : new int[] {19, 103, 1019}) { + List<Entry<Key,Value>> sampleDataLG1 = new ArrayList<>(); + List<Entry<Key,Value>> sampleDataLG2 = new ArrayList<>(); + + ConfigurationCopy sampleConf = new ConfigurationCopy(conf == null ? AccumuloConfiguration.getDefaultConfiguration() : conf); + sampleConf.set(Property.TABLE_SAMPLER, RowSampler.class.getName()); + sampleConf.set(Property.TABLE_SAMPLER_OPTS + "hasher", "murmur3_32"); + sampleConf.set(Property.TABLE_SAMPLER_OPTS + "modulus", modulus + ""); + + Sampler sampler = SamplerFactory.newSampler(SamplerConfigurationImpl.newSamplerConfig(sampleConf), sampleConf); + + TestRFile trf = new TestRFile(sampleConf); + + trf.openWriter(false, 1000); + + trf.writer.startNewLocalityGroup("meta-lg", ncfs("metaA", "metaB")); + for (int r = 0; r < num; r++) { + String row = String.format("r%06d", r); + Key k1 = new Key(row, "metaA", "q9", 7); + Key k2 = new Key(row, "metaB", "q8", 7); + Key k3 = new Key(row, "metaB", "qA", 7); + + Value v1 = new Value(("" + r).getBytes()); + Value v2 = new Value(("" + r * 93).getBytes()); + Value v3 = new Value(("" + r * 113).getBytes()); + + if (sampler.accept(k1)) { + sampleDataLG1.add(new AbstractMap.SimpleImmutableEntry<>(k1, v1)); + sampleDataLG1.add(new AbstractMap.SimpleImmutableEntry<>(k2, v2)); + sampleDataLG1.add(new AbstractMap.SimpleImmutableEntry<>(k3, v3)); + } + + trf.writer.append(k1, v1); + trf.writer.append(k2, v2); + trf.writer.append(k3, v3); + } + + trf.writer.startDefaultLocalityGroup(); + + for (int r = 0; r < num; r++) { + String row = String.format("r%06d", r); + Key k1 = new Key(row, "dataA", "q9", 7); + + Value v1 = new Value(("" + r).getBytes()); + + if (sampler.accept(k1)) { + sampleDataLG2.add(new AbstractMap.SimpleImmutableEntry<>(k1, v1)); + } + + trf.writer.append(k1, v1); + } + + trf.closeWriter(); + + Assert.assertTrue(sampleDataLG1.size() > 0); + Assert.assertTrue(sampleDataLG2.size() > 0); + + trf.openReader(false); + FileSKVIterator sample = trf.reader.getSample(SamplerConfigurationImpl.newSamplerConfig(sampleConf)); + + checkSample(sample, sampleDataLG1, ncfs("metaA", "metaB"), true); + checkSample(sample, sampleDataLG1, ncfs("metaA"), true); + checkSample(sample, sampleDataLG1, ncfs("metaB"), true); + checkSample(sample, sampleDataLG1, ncfs("dataA"), false); + + checkSample(sample, sampleDataLG2, ncfs("metaA", "metaB"), false); + checkSample(sample, sampleDataLG2, ncfs("dataA"), true); + + ArrayList<Entry<Key,Value>> allSampleData = new ArrayList<>(); + allSampleData.addAll(sampleDataLG1); + allSampleData.addAll(sampleDataLG2); + + Collections.sort(allSampleData, new Comparator<Entry<Key,Value>>() { + @Override + public int compare(Entry<Key,Value> o1, Entry<Key,Value> o2) { + return o1.getKey().compareTo(o2.getKey()); + } + }); + + checkSample(sample, allSampleData, ncfs("dataA", "metaA"), true); + checkSample(sample, allSampleData, EMPTY_COL_FAMS, false); + + trf.closeReader(); + } + } + } + + @Test + public void testEncSample() throws IOException { + conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); + testSample(); + testSampleLG(); + conf = null; + } + @Test public void testBigKeys() throws IOException { // this test ensures that big keys do not end up index http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/file/rfile/RelativeKeyTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/iterators/IteratorUtilTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/iterators/system/MultiIteratorTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/iterators/system/MultiIteratorTest.java index 8949c92,d599674..b2de6cb --- a/core/src/test/java/org/apache/accumulo/core/iterators/system/MultiIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/system/MultiIteratorTest.java @@@ -344,7 -344,7 +344,8 @@@ public class MultiIteratorTest extends List<SortedKeyValueIterator<Key,Value>> skvil = new ArrayList<>(1); skvil.add(new SortedMapIterator(tm1)); - KeyExtent extent = new KeyExtent("tablename", nr(1), nr(0)); - KeyExtent extent = new KeyExtent(new Text("tablename"), newRow(1), newRow(0)); ++ KeyExtent extent = new KeyExtent("tablename", newRow(1), newRow(0)); ++ MultiIterator mi = new MultiIterator(skvil, extent); Range r1 = new Range((Text) null, (Text) null); http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java index 4ec0269,e16c912..15c424b --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java @@@ -54,9 -63,22 +54,9 @@@ public class RowDeletingIteratorTest ex public boolean isFullMajorCompaction() { return fmc; } - - @Override - public SortedKeyValueIterator<Key,Value> reserveMapFileReader(String mapFileName) throws IOException { - return null; - } - - @Override - public void registerSideChannel(SortedKeyValueIterator<Key,Value> iter) {} - - @Override - public Authorizations getAuthorizations() { - return null; - } } - Key nk(String row, String cf, String cq, long time) { + Key newKey(String row, String cf, String cq, long time) { return new Key(new Text(row), new Text(cf), new Text(cq), time); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/server/tserver/src/test/java/org/apache/accumulo/tserver/CheckTabletMetadataTest.java ---------------------------------------------------------------------- diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/CheckTabletMetadataTest.java index f474972,8d8767f..221e921 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/CheckTabletMetadataTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/CheckTabletMetadataTest.java @@@ -89,17 -89,17 +89,17 @@@ public class CheckTabletMetadataTest assertFail(tabletMeta, ke, new TServerInstance("127.0.0.2:9997", 4)); assertFail(tabletMeta, ke, new TServerInstance("127.0.0.2:9997", 5)); - assertFail(tabletMeta, new KeyExtent(new Text("1"), null, new Text("m")), tsi); + assertFail(tabletMeta, new KeyExtent("1", null, new Text("m")), tsi); - assertFail(tabletMeta, new KeyExtent(new Text("1"), new Text("r"), new Text("m")), tsi); + assertFail(tabletMeta, new KeyExtent("1", new Text("r"), new Text("m")), tsi); - assertFail(tabletMeta, ke, tsi, nk("1<", TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN)); + assertFail(tabletMeta, ke, tsi, newKey("1<", TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN)); - assertFail(tabletMeta, ke, tsi, nk("1<", TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN)); + assertFail(tabletMeta, ke, tsi, newKey("1<", TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN)); - assertFail(tabletMeta, ke, tsi, nk("1<", TabletsSection.ServerColumnFamily.TIME_COLUMN)); + assertFail(tabletMeta, ke, tsi, newKey("1<", TabletsSection.ServerColumnFamily.TIME_COLUMN)); - assertFail(tabletMeta, ke, tsi, nk("1<", TabletsSection.FutureLocationColumnFamily.NAME, "4")); + assertFail(tabletMeta, ke, tsi, newKey("1<", TabletsSection.FutureLocationColumnFamily.NAME, "4")); TreeMap<Key,Value> copy = new TreeMap<>(tabletMeta); put(copy, "1<", TabletsSection.CurrentLocationColumnFamily.NAME, "4", "127.0.0.1:9997"); http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java ---------------------------------------------------------------------- diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java index f813d42,af77150..2733efd --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java @@@ -208,11 -138,11 +208,11 @@@ public class InMemoryMapTest mutate(imm, "r3", "foo:cq1", 3, "bar9"); mutate(imm, "r3", "foo:cq1", 3, "bara"); - MemoryIterator ski2 = imm.skvIterator(); + MemoryIterator ski2 = imm.skvIterator(null); ski1.seek(new Range(new Text("r1")), LocalityGroupUtil.EMPTY_CF_SET, false); - ae(ski1, "r1", "foo:cq1", 3, "bar2"); - ae(ski1, "r1", "foo:cq1", 3, "bar1"); + testAndCallNext(ski1, "r1", "foo:cq1", 3, "bar2"); + testAndCallNext(ski1, "r1", "foo:cq1", 3, "bar1"); assertFalse(ski1.hasTop()); ski2.seek(new Range(new Text("r3")), LocalityGroupUtil.EMPTY_CF_SET, false); @@@ -281,9 -192,9 +281,9 @@@ mutate(imm, "r1", "foo:cq1", 3, "bar2"); mutate(imm, "r1", "foo:cq1", 3, "bar3"); - MemoryIterator ski1 = imm.skvIterator(); + MemoryIterator ski1 = imm.skvIterator(null); ski1.seek(new Range(new Text("r1")), LocalityGroupUtil.EMPTY_CF_SET, false); - ae(ski1, "r1", "foo:cq1", 3, "bar3"); + testAndCallNext(ski1, "r1", "foo:cq1", 3, "bar3"); imm.delete(0); @@@ -299,9 -210,9 +299,9 @@@ mutate(imm, "r1", "foo:cq2", 3, "bar2"); mutate(imm, "r1", "foo:cq3", 3, "bar3"); - ski1 = imm.skvIterator(); + ski1 = imm.skvIterator(null); ski1.seek(new Range(new Text("r1")), LocalityGroupUtil.EMPTY_CF_SET, false); - ae(ski1, "r1", "foo:cq1", 3, "bar1"); + testAndCallNext(ski1, "r1", "foo:cq1", 3, "bar1"); imm.delete(0); @@@ -325,13 -236,13 +325,13 @@@ mutate(imm, "r1", "foo:cq5", 3, "bar5"); - SortedKeyValueIterator<Key,Value> dc = ski1.deepCopy(null); + SortedKeyValueIterator<Key,Value> dc = ski1.deepCopy(new SampleIE()); - ski1.seek(new Range(nk("r1", "foo:cq1", 3), null), LocalityGroupUtil.EMPTY_CF_SET, false); - ae(ski1, "r1", "foo:cq1", 3, "bar1"); + ski1.seek(new Range(newKey("r1", "foo:cq1", 3), null), LocalityGroupUtil.EMPTY_CF_SET, false); + testAndCallNext(ski1, "r1", "foo:cq1", 3, "bar1"); - dc.seek(new Range(nk("r1", "foo:cq2", 3), null), LocalityGroupUtil.EMPTY_CF_SET, false); - ae(dc, "r1", "foo:cq2", 3, "bar2"); + dc.seek(new Range(newKey("r1", "foo:cq2", 3), null), LocalityGroupUtil.EMPTY_CF_SET, false); + testAndCallNext(dc, "r1", "foo:cq2", 3, "bar2"); imm.delete(0); @@@ -462,13 -373,13 +462,13 @@@ mutate(imm, "r1", "foo:cq3", 3, "bar3"); mutate(imm, "r1", "foo:cq4", 3, "bar4"); - MemoryIterator skvi1 = imm.skvIterator(); + MemoryIterator skvi1 = imm.skvIterator(null); - skvi1.seek(new Range(nk("r1", "foo:cq3", 3), null), LocalityGroupUtil.EMPTY_CF_SET, false); - ae(skvi1, "r1", "foo:cq3", 3, "bar3"); + skvi1.seek(new Range(newKey("r1", "foo:cq3", 3), null), LocalityGroupUtil.EMPTY_CF_SET, false); + testAndCallNext(skvi1, "r1", "foo:cq3", 3, "bar3"); - skvi1.seek(new Range(nk("r1", "foo:cq1", 3), null), LocalityGroupUtil.EMPTY_CF_SET, false); - ae(skvi1, "r1", "foo:cq1", 3, "bar1"); + skvi1.seek(new Range(newKey("r1", "foo:cq1", 3), null), LocalityGroupUtil.EMPTY_CF_SET, false); + testAndCallNext(skvi1, "r1", "foo:cq1", 3, "bar1"); } @@@ -481,10 -392,10 +481,10 @@@ m.put(new Text("foo"), new Text("cq"), 3, new Value("v2".getBytes())); imm.mutate(Collections.singletonList(m)); - MemoryIterator skvi1 = imm.skvIterator(); + MemoryIterator skvi1 = imm.skvIterator(null); skvi1.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false); - ae(skvi1, "r1", "foo:cq", 3, "v2"); - ae(skvi1, "r1", "foo:cq", 3, "v1"); + testAndCallNext(skvi1, "r1", "foo:cq", 3, "v2"); + testAndCallNext(skvi1, "r1", "foo:cq", 3, "v1"); } private static final Logger log = Logger.getLogger(InMemoryMapTest.class); @@@ -586,259 -497,11 +586,259 @@@ // seekLocalityGroups(iter1.deepCopy(null)); } + @Test + public void testSample() throws Exception { + + SamplerConfigurationImpl sampleConfig = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "7")); + Sampler sampler = SamplerFactory.newSampler(sampleConfig, DefaultConfiguration.getInstance()); + + ConfigurationCopy config1 = newConfig(tempFolder.newFolder().getAbsolutePath()); + for (Entry<String,String> entry : sampleConfig.toTablePropertiesMap().entrySet()) { + config1.set(entry.getKey(), entry.getValue()); + } + + ConfigurationCopy config2 = newConfig(tempFolder.newFolder().getAbsolutePath()); + config2.set(Property.TABLE_LOCALITY_GROUP_PREFIX + "lg1", LocalityGroupUtil.encodeColumnFamilies(toTextSet("cf2"))); + config2.set(Property.TABLE_LOCALITY_GROUPS.getKey(), "lg1"); + for (Entry<String,String> entry : sampleConfig.toTablePropertiesMap().entrySet()) { + config2.set(entry.getKey(), entry.getValue()); + } + + for (ConfigurationCopy config : Arrays.asList(config1, config2)) { + + InMemoryMap imm = new InMemoryMap(config); + + TreeMap<Key,Value> expectedSample = new TreeMap<>(); + TreeMap<Key,Value> expectedAll = new TreeMap<>(); + TreeMap<Key,Value> expectedNone = new TreeMap<>(); + + MemoryIterator iter0 = imm.skvIterator(sampleConfig); + + for (int r = 0; r < 100; r++) { + String row = String.format("r%06d", r); + mutate(imm, row, "cf1:cq1", 5, "v" + (2 * r), sampler, expectedSample, expectedAll); + mutate(imm, row, "cf2:cq2", 5, "v" + ((2 * r) + 1), sampler, expectedSample, expectedAll); + } + + assertTrue(expectedSample.size() > 0); + + MemoryIterator iter1 = imm.skvIterator(sampleConfig); + MemoryIterator iter2 = imm.skvIterator(null); + SortedKeyValueIterator<Key,Value> iter0dc1 = iter0.deepCopy(new SampleIE()); + SortedKeyValueIterator<Key,Value> iter0dc2 = iter0.deepCopy(new SampleIE(sampleConfig)); + SortedKeyValueIterator<Key,Value> iter1dc1 = iter1.deepCopy(new SampleIE()); + SortedKeyValueIterator<Key,Value> iter1dc2 = iter1.deepCopy(new SampleIE(sampleConfig)); + SortedKeyValueIterator<Key,Value> iter2dc1 = iter2.deepCopy(new SampleIE()); + SortedKeyValueIterator<Key,Value> iter2dc2 = iter2.deepCopy(new SampleIE(sampleConfig)); + + assertEquals(expectedNone, readAll(iter0)); + assertEquals(expectedNone, readAll(iter0dc1)); + assertEquals(expectedNone, readAll(iter0dc2)); + assertEquals(expectedSample, readAll(iter1)); + assertEquals(expectedAll, readAll(iter2)); + assertEquals(expectedAll, readAll(iter1dc1)); + assertEquals(expectedAll, readAll(iter2dc1)); + assertEquals(expectedSample, readAll(iter1dc2)); + assertEquals(expectedSample, readAll(iter2dc2)); + + imm.delete(0); + + assertEquals(expectedNone, readAll(iter0)); + assertEquals(expectedNone, readAll(iter0dc1)); + assertEquals(expectedNone, readAll(iter0dc2)); + assertEquals(expectedSample, readAll(iter1)); + assertEquals(expectedAll, readAll(iter2)); + assertEquals(expectedAll, readAll(iter1dc1)); + assertEquals(expectedAll, readAll(iter2dc1)); + assertEquals(expectedSample, readAll(iter1dc2)); + assertEquals(expectedSample, readAll(iter2dc2)); + + SortedKeyValueIterator<Key,Value> iter0dc3 = iter0.deepCopy(new SampleIE()); + SortedKeyValueIterator<Key,Value> iter0dc4 = iter0.deepCopy(new SampleIE(sampleConfig)); + SortedKeyValueIterator<Key,Value> iter1dc3 = iter1.deepCopy(new SampleIE()); + SortedKeyValueIterator<Key,Value> iter1dc4 = iter1.deepCopy(new SampleIE(sampleConfig)); + SortedKeyValueIterator<Key,Value> iter2dc3 = iter2.deepCopy(new SampleIE()); + SortedKeyValueIterator<Key,Value> iter2dc4 = iter2.deepCopy(new SampleIE(sampleConfig)); + + assertEquals(expectedNone, readAll(iter0dc3)); + assertEquals(expectedNone, readAll(iter0dc4)); + assertEquals(expectedAll, readAll(iter1dc3)); + assertEquals(expectedAll, readAll(iter2dc3)); + assertEquals(expectedSample, readAll(iter1dc4)); + assertEquals(expectedSample, readAll(iter2dc4)); + + iter1.close(); + iter2.close(); + } + } + + @Test + public void testInterruptingSample() throws Exception { + runInterruptSampleTest(false, false, false); + runInterruptSampleTest(false, true, false); + runInterruptSampleTest(true, false, false); + runInterruptSampleTest(true, true, false); + runInterruptSampleTest(true, true, true); + } + + private void runInterruptSampleTest(boolean deepCopy, boolean delete, boolean dcAfterDelete) throws Exception { + SamplerConfigurationImpl sampleConfig1 = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "2")); + Sampler sampler = SamplerFactory.newSampler(sampleConfig1, DefaultConfiguration.getInstance()); + + ConfigurationCopy config1 = newConfig(tempFolder.newFolder().getAbsolutePath()); + for (Entry<String,String> entry : sampleConfig1.toTablePropertiesMap().entrySet()) { + config1.set(entry.getKey(), entry.getValue()); + } + + InMemoryMap imm = new InMemoryMap(config1); + + TreeMap<Key,Value> expectedSample = new TreeMap<>(); + TreeMap<Key,Value> expectedAll = new TreeMap<>(); + + for (int r = 0; r < 1000; r++) { + String row = String.format("r%06d", r); + mutate(imm, row, "cf1:cq1", 5, "v" + (2 * r), sampler, expectedSample, expectedAll); + mutate(imm, row, "cf2:cq2", 5, "v" + ((2 * r) + 1), sampler, expectedSample, expectedAll); + } + + assertTrue(expectedSample.size() > 0); + + MemoryIterator miter = imm.skvIterator(sampleConfig1); + AtomicBoolean iFlag = new AtomicBoolean(false); + miter.setInterruptFlag(iFlag); + SortedKeyValueIterator<Key,Value> iter = miter; + + if (delete && !dcAfterDelete) { + imm.delete(0); + } + + if (deepCopy) { + iter = iter.deepCopy(new SampleIE(sampleConfig1)); + } + + if (delete && dcAfterDelete) { + imm.delete(0); + } + + assertEquals(expectedSample, readAll(iter)); + iFlag.set(true); + try { + readAll(iter); + Assert.fail(); + } catch (IterationInterruptedException iie) {} + + miter.close(); + } + + private void mutate(InMemoryMap imm, String row, String cols, int ts, String val, Sampler sampler, TreeMap<Key,Value> expectedSample, + TreeMap<Key,Value> expectedAll) { + mutate(imm, row, cols, ts, val); + Key k1 = nk(row, cols, ts); + if (sampler.accept(k1)) { + expectedSample.put(k1, new Value(val.getBytes())); + } + expectedAll.put(k1, new Value(val.getBytes())); + } + + @Test(expected = SampleNotPresentException.class) + public void testDifferentSampleConfig() throws Exception { + SamplerConfigurationImpl sampleConfig = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "7")); + + ConfigurationCopy config1 = newConfig(tempFolder.newFolder().getAbsolutePath()); + for (Entry<String,String> entry : sampleConfig.toTablePropertiesMap().entrySet()) { + config1.set(entry.getKey(), entry.getValue()); + } + + InMemoryMap imm = new InMemoryMap(config1); + + mutate(imm, "r", "cf:cq", 5, "b"); + + SamplerConfigurationImpl sampleConfig2 = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "9")); + MemoryIterator iter = imm.skvIterator(sampleConfig2); + iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false); + } + + @Test(expected = SampleNotPresentException.class) + public void testNoSampleConfig() throws Exception { + InMemoryMap imm = newInMemoryMap(false, tempFolder.newFolder().getAbsolutePath()); + + mutate(imm, "r", "cf:cq", 5, "b"); + + SamplerConfigurationImpl sampleConfig2 = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "9")); + MemoryIterator iter = imm.skvIterator(sampleConfig2); + iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false); + } + + @Test + public void testEmptyNoSampleConfig() throws Exception { + InMemoryMap imm = newInMemoryMap(false, tempFolder.newFolder().getAbsolutePath()); + + SamplerConfigurationImpl sampleConfig2 = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "9")); + + // when in mem map is empty should be able to get sample iterator with any sample config + MemoryIterator iter = imm.skvIterator(sampleConfig2); + iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false); + Assert.assertFalse(iter.hasTop()); + } + + @Test + public void testDeferredSamplerCreation() throws Exception { + SamplerConfigurationImpl sampleConfig1 = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "9")); + + ConfigurationCopy config1 = newConfig(tempFolder.newFolder().getAbsolutePath()); + for (Entry<String,String> entry : sampleConfig1.toTablePropertiesMap().entrySet()) { + config1.set(entry.getKey(), entry.getValue()); + } + + InMemoryMap imm = new InMemoryMap(config1); + + // change sampler config after creating in mem map. + SamplerConfigurationImpl sampleConfig2 = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "7")); + for (Entry<String,String> entry : sampleConfig2.toTablePropertiesMap().entrySet()) { + config1.set(entry.getKey(), entry.getValue()); + } + + TreeMap<Key,Value> expectedSample = new TreeMap<>(); + TreeMap<Key,Value> expectedAll = new TreeMap<>(); + Sampler sampler = SamplerFactory.newSampler(sampleConfig2, config1); + + for (int i = 0; i < 100; i++) { + mutate(imm, "r" + i, "cf:cq", 5, "v" + i, sampler, expectedSample, expectedAll); + } + + MemoryIterator iter = imm.skvIterator(sampleConfig2); + iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false); + Assert.assertEquals(expectedSample, readAll(iter)); + + SortedKeyValueIterator<Key,Value> dc = iter.deepCopy(new SampleIE(sampleConfig2)); + dc.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false); + Assert.assertEquals(expectedSample, readAll(dc)); + + iter = imm.skvIterator(null); + iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false); + Assert.assertEquals(expectedAll, readAll(iter)); + + iter = imm.skvIterator(sampleConfig1); + thrown.expect(SampleNotPresentException.class); + iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false); + } + + private TreeMap<Key,Value> readAll(SortedKeyValueIterator<Key,Value> iter) throws IOException { + iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false); + + TreeMap<Key,Value> actual = new TreeMap<>(); + while (iter.hasTop()) { + actual.put(iter.getTopKey(), iter.getTopValue()); + iter.next(); + } + return actual; + } + private void seekLocalityGroups(SortedKeyValueIterator<Key,Value> iter1) throws IOException { iter1.seek(new Range(), newCFSet("cf1"), true); - ae(iter1, "r1", "cf1:x", 2, "1"); - ae(iter1, "r1", "cf1:y", 2, "2"); - ae(iter1, "r2", "cf2:x", 3, "5"); + testAndCallNext(iter1, "r1", "cf1:x", 2, "1"); + testAndCallNext(iter1, "r1", "cf1:y", 2, "2"); + testAndCallNext(iter1, "r2", "cf2:x", 3, "5"); assertFalse(iter1.hasTop()); iter1.seek(new Range("r2", "r4"), newCFSet("cf1"), true); http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/test/src/main/java/org/apache/accumulo/test/functional/NativeMapIT.java ---------------------------------------------------------------------- diff --cc test/src/main/java/org/apache/accumulo/test/functional/NativeMapIT.java index ea3ba66,0000000..fc9e8d1 mode 100644,000000..100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/NativeMapIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/NativeMapIT.java @@@ -1,616 -1,0 +1,617 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.functional; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.NoSuchElementException; +import java.util.Random; +import java.util.TreeMap; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.util.Pair; +import org.apache.accumulo.test.categories.SunnyDayTests; +import org.apache.accumulo.tserver.NativeMap; +import org.apache.hadoop.io.Text; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SunnyDayTests.class) +public class NativeMapIT { + - private Key nk(int r) { ++ private Key newKey(int r) { + return new Key(new Text(String.format("r%09d", r))); + } + - private Key nk(int r, int cf, int cq, int cv, int ts, boolean deleted) { ++ private Key newKey(int r, int cf, int cq, int cv, int ts, boolean deleted) { + Key k = new Key(new Text(String.format("r%09d", r)), new Text(String.format("cf%09d", cf)), new Text(String.format("cq%09d", cq)), new Text(String.format( + "cv%09d", cv)), ts); + + k.setDeleted(deleted); + + return k; + } + - private Value nv(int v) { ++ private Value newValue(int v) { + return new Value(String.format("r%09d", v).getBytes(UTF_8)); + } + + public static File nativeMapLocation() { + File projectDir = new File(System.getProperty("user.dir")).getParentFile(); + File nativeMapDir = new File(projectDir, "server/native/target/accumulo-native-" + Constants.VERSION + "/accumulo-native-" + Constants.VERSION); + return nativeMapDir; + } + + @BeforeClass + public static void setUp() { + NativeMap.loadNativeLib(Collections.singletonList(nativeMapLocation())); + } + + private void verifyIterator(int start, int end, int valueOffset, Iterator<Entry<Key,Value>> iter) { + for (int i = start; i <= end; i++) { + assertTrue(iter.hasNext()); + Entry<Key,Value> entry = iter.next(); - assertEquals(nk(i), entry.getKey()); - assertEquals(nv(i + valueOffset), entry.getValue()); ++ assertEquals(newKey(i), entry.getKey()); ++ assertEquals(newValue(i + valueOffset), entry.getValue()); + } + + assertFalse(iter.hasNext()); + } + + private void insertAndVerify(NativeMap nm, int start, int end, int valueOffset) { + for (int i = start; i <= end; i++) { - nm.put(nk(i), nv(i + valueOffset)); ++ nm.put(newKey(i), newValue(i + valueOffset)); + } + + for (int i = start; i <= end; i++) { - Value v = nm.get(nk(i)); ++ Value v = nm.get(newKey(i)); + assertNotNull(v); - assertEquals(nv(i + valueOffset), v); ++ assertEquals(newValue(i + valueOffset), v); + - Iterator<Entry<Key,Value>> iter2 = nm.iterator(nk(i)); ++ Iterator<Entry<Key,Value>> iter2 = nm.iterator(newKey(i)); + assertTrue(iter2.hasNext()); + Entry<Key,Value> entry = iter2.next(); - assertEquals(nk(i), entry.getKey()); - assertEquals(nv(i + valueOffset), entry.getValue()); ++ assertEquals(newKey(i), entry.getKey()); ++ assertEquals(newValue(i + valueOffset), entry.getValue()); + } + - assertNull(nm.get(nk(start - 1))); ++ assertNull(nm.get(newKey(start - 1))); + - assertNull(nm.get(nk(end + 1))); ++ assertNull(nm.get(newKey(end + 1))); + + Iterator<Entry<Key,Value>> iter = nm.iterator(); + verifyIterator(start, end, valueOffset, iter); + + for (int i = start; i <= end; i++) { - iter = nm.iterator(nk(i)); ++ iter = nm.iterator(newKey(i)); + verifyIterator(i, end, valueOffset, iter); + + // lookup nonexistant key that falls after existing key - iter = nm.iterator(nk(i, 1, 1, 1, 1, false)); ++ iter = nm.iterator(newKey(i, 1, 1, 1, 1, false)); + verifyIterator(i + 1, end, valueOffset, iter); + } + + assertEquals(end - start + 1, nm.size()); + } + + private void insertAndVerifyExhaustive(NativeMap nm, int num, int run) { + for (int i = 0; i < num; i++) { + for (int j = 0; j < num; j++) { + for (int k = 0; k < num; k++) { + for (int l = 0; l < num; l++) { + for (int ts = 0; ts < num; ts++) { - Key key = nk(i, j, k, l, ts, true); ++ Key key = newKey(i, j, k, l, ts, true); + Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8)); + + nm.put(key, value); + - key = nk(i, j, k, l, ts, false); ++ key = newKey(i, j, k, l, ts, false); + value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8)); + + nm.put(key, value); + } + } + } + } + } + + Iterator<Entry<Key,Value>> iter = nm.iterator(); + + for (int i = 0; i < num; i++) { + for (int j = 0; j < num; j++) { + for (int k = 0; k < num; k++) { + for (int l = 0; l < num; l++) { + for (int ts = num - 1; ts >= 0; ts--) { - Key key = nk(i, j, k, l, ts, true); ++ Key key = newKey(i, j, k, l, ts, true); + Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8)); + + assertTrue(iter.hasNext()); + Entry<Key,Value> entry = iter.next(); + assertEquals(key, entry.getKey()); + assertEquals(value, entry.getValue()); + - key = nk(i, j, k, l, ts, false); ++ key = newKey(i, j, k, l, ts, false); + value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8)); + + assertTrue(iter.hasNext()); + entry = iter.next(); + assertEquals(key, entry.getKey()); + assertEquals(value, entry.getValue()); + } + } + } + } + } + + assertFalse(iter.hasNext()); + + for (int i = 0; i < num; i++) { + for (int j = 0; j < num; j++) { + for (int k = 0; k < num; k++) { + for (int l = 0; l < num; l++) { + for (int ts = 0; ts < num; ts++) { - Key key = nk(i, j, k, l, ts, true); ++ Key key = newKey(i, j, k, l, ts, true); + Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8)); + + assertEquals(value, nm.get(key)); + + Iterator<Entry<Key,Value>> iter2 = nm.iterator(key); + assertTrue(iter2.hasNext()); + Entry<Key,Value> entry = iter2.next(); + assertEquals(key, entry.getKey()); + assertEquals(value, entry.getValue()); + - key = nk(i, j, k, l, ts, false); ++ key = newKey(i, j, k, l, ts, false); + value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8)); + + assertEquals(value, nm.get(key)); + + Iterator<Entry<Key,Value>> iter3 = nm.iterator(key); + assertTrue(iter3.hasNext()); + Entry<Key,Value> entry2 = iter3.next(); + assertEquals(key, entry2.getKey()); + assertEquals(value, entry2.getValue()); + } + } + } + } + } + + assertEquals(num * num * num * num * num * 2, nm.size()); + } + + @Test + public void test1() { + NativeMap nm = new NativeMap(); + Iterator<Entry<Key,Value>> iter = nm.iterator(); + assertFalse(iter.hasNext()); + nm.delete(); + } + + @Test + public void test2() { + NativeMap nm = new NativeMap(); + + insertAndVerify(nm, 1, 10, 0); + insertAndVerify(nm, 1, 10, 1); + insertAndVerify(nm, 1, 10, 2); + + nm.delete(); + } + + @Test + public void test4() { + NativeMap nm = new NativeMap(); + + insertAndVerifyExhaustive(nm, 3, 0); + insertAndVerifyExhaustive(nm, 3, 1); + + nm.delete(); + } + + @Test + public void test5() { + NativeMap nm = new NativeMap(); + + insertAndVerify(nm, 1, 10, 0); + + Iterator<Entry<Key,Value>> iter = nm.iterator(); + iter.next(); + + nm.delete(); + + try { - nm.put(nk(1), nv(1)); ++ nm.put(newKey(1), newValue(1)); + assertTrue(false); + } catch (IllegalStateException e) { + + } + + try { - nm.get(nk(1)); ++ nm.get(newKey(1)); + assertTrue(false); + } catch (IllegalStateException e) { + + } + + try { + nm.iterator(); + assertTrue(false); + } catch (IllegalStateException e) { + + } + + try { - nm.iterator(nk(1)); ++ nm.iterator(newKey(1)); + assertTrue(false); + } catch (IllegalStateException e) { + + } + + try { + nm.size(); + assertTrue(false); + } catch (IllegalStateException e) { + + } + + try { + iter.next(); + assertTrue(false); + } catch (IllegalStateException e) { + + } + + } + + @Test + public void test7() { + NativeMap nm = new NativeMap(); + + insertAndVerify(nm, 1, 10, 0); + + nm.delete(); + + try { + nm.delete(); + assertTrue(false); + } catch (IllegalStateException e) { + + } + } + + @Test + public void test8() { + // test verifies that native map sorts keys sharing some common prefix properly + + NativeMap nm = new NativeMap(); + + TreeMap<Key,Value> tm = new TreeMap<>(); + + tm.put(new Key(new Text("fo")), new Value(new byte[] {'0'})); + tm.put(new Key(new Text("foo")), new Value(new byte[] {'1'})); + tm.put(new Key(new Text("foo1")), new Value(new byte[] {'2'})); + tm.put(new Key(new Text("foo2")), new Value(new byte[] {'3'})); + + for (Entry<Key,Value> entry : tm.entrySet()) { + nm.put(entry.getKey(), entry.getValue()); + } + + Iterator<Entry<Key,Value>> iter = nm.iterator(); + + for (Entry<Key,Value> entry : tm.entrySet()) { + assertTrue(iter.hasNext()); + Entry<Key,Value> entry2 = iter.next(); + + assertEquals(entry.getKey(), entry2.getKey()); + assertEquals(entry.getValue(), entry2.getValue()); + } + + assertFalse(iter.hasNext()); + + nm.delete(); + } + + @Test + public void test9() { + NativeMap nm = new NativeMap(); + + Iterator<Entry<Key,Value>> iter = nm.iterator(); + + try { + iter.next(); + assertTrue(false); + } catch (NoSuchElementException e) { + + } + + insertAndVerify(nm, 1, 1, 0); + + iter = nm.iterator(); + iter.next(); + + try { + iter.next(); + assertTrue(false); + } catch (NoSuchElementException e) { + + } + + nm.delete(); + } + + @Test + public void test10() { + int start = 1; + int end = 10000; + + NativeMap nm = new NativeMap(); + for (int i = start; i <= end; i++) { - nm.put(nk(i), nv(i)); ++ nm.put(newKey(i), newValue(i)); + } + + long mem1 = nm.getMemoryUsed(); + + for (int i = start; i <= end; i++) { - nm.put(nk(i), nv(i)); ++ nm.put(newKey(i), newValue(i)); + } + + long mem2 = nm.getMemoryUsed(); + + if (mem1 != mem2) { + throw new RuntimeException("Memory changed after inserting duplicate data " + mem1 + " " + mem2); + } + + for (int i = start; i <= end; i++) { - nm.put(nk(i), nv(i)); ++ nm.put(newKey(i), newValue(i)); + } + + long mem3 = nm.getMemoryUsed(); + + if (mem1 != mem3) { + throw new RuntimeException("Memory changed after inserting duplicate data " + mem1 + " " + mem3); + } + + byte bigrow[] = new byte[1000000]; + byte bigvalue[] = new byte[bigrow.length]; + + for (int i = 0; i < bigrow.length; i++) { + bigrow[i] = (byte) (0xff & (i % 256)); + bigvalue[i] = bigrow[i]; + } + + nm.put(new Key(new Text(bigrow)), new Value(bigvalue)); + + long mem4 = nm.getMemoryUsed(); + + Value val = nm.get(new Key(new Text(bigrow))); + if (val == null || !val.equals(new Value(bigvalue))) { + throw new RuntimeException("Did not get expected big value"); + } + + nm.put(new Key(new Text(bigrow)), new Value(bigvalue)); + + long mem5 = nm.getMemoryUsed(); + + if (mem4 != mem5) { + throw new RuntimeException("Memory changed after inserting duplicate data " + mem4 + " " + mem5); + } + + val = nm.get(new Key(new Text(bigrow))); + if (val == null || !val.equals(new Value(bigvalue))) { + throw new RuntimeException("Did not get expected big value"); + } + + nm.delete(); + } + + // random length random field - private static byte[] rlrf(Random r, int maxLen) { ++ private static byte[] getRandomBytes(Random r, int maxLen) { + int len = r.nextInt(maxLen); + + byte f[] = new byte[len]; + r.nextBytes(f); + + return f; + } + + @Test + public void test11() { + NativeMap nm = new NativeMap(); + + // insert things with varying field sizes and value sizes + + // generate random data + Random r = new Random(75); + + ArrayList<Pair<Key,Value>> testData = new ArrayList<>(); + + for (int i = 0; i < 100000; i++) { + - Key k = new Key(rlrf(r, 97), rlrf(r, 13), rlrf(r, 31), rlrf(r, 11), (r.nextLong() & 0x7fffffffffffffffl), false, false); - Value v = new Value(rlrf(r, 511)); ++ Key k = new Key(getRandomBytes(r, 97), getRandomBytes(r, 13), getRandomBytes(r, 31), getRandomBytes(r, 11), (r.nextLong() & 0x7fffffffffffffffl), false, ++ false); ++ Value v = new Value(getRandomBytes(r, 511)); + + testData.add(new Pair<>(k, v)); + } + + // insert unsorted data + for (Pair<Key,Value> pair : testData) { + nm.put(pair.getFirst(), pair.getSecond()); + } + + for (int i = 0; i < 2; i++) { + + // sort data + Collections.sort(testData, new Comparator<Pair<Key,Value>>() { + @Override + public int compare(Pair<Key,Value> o1, Pair<Key,Value> o2) { + return o1.getFirst().compareTo(o2.getFirst()); + } + }); + + // verify + Iterator<Entry<Key,Value>> iter1 = nm.iterator(); + Iterator<Pair<Key,Value>> iter2 = testData.iterator(); + + while (iter1.hasNext() && iter2.hasNext()) { + Entry<Key,Value> e = iter1.next(); + Pair<Key,Value> p = iter2.next(); + + if (!e.getKey().equals(p.getFirst())) + throw new RuntimeException("Keys not equal"); + + if (!e.getValue().equals(p.getSecond())) + throw new RuntimeException("Values not equal"); + } + + if (iter1.hasNext()) + throw new RuntimeException("Not all of native map consumed"); + + if (iter2.hasNext()) + throw new RuntimeException("Not all of test data consumed"); + + System.out.println("test 11 nm mem " + nm.getMemoryUsed()); + + // insert data again w/ different value + Collections.shuffle(testData, r); + // insert unsorted data + for (Pair<Key,Value> pair : testData) { - pair.getSecond().set(rlrf(r, 511)); ++ pair.getSecond().set(getRandomBytes(r, 511)); + nm.put(pair.getFirst(), pair.getSecond()); + } + } + + nm.delete(); + } + + @Test + public void testBinary() { + NativeMap nm = new NativeMap(); + + byte emptyBytes[] = new byte[0]; + + for (int i = 0; i < 256; i++) { + for (int j = 0; j < 256; j++) { + byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)}; + byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)}; + + Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1); + Value v = new Value(data); + + nm.put(k, v); + } + } + + Iterator<Entry<Key,Value>> iter = nm.iterator(); + for (int i = 0; i < 256; i++) { + for (int j = 0; j < 256; j++) { + byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)}; + byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)}; + + Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1); + Value v = new Value(data); + + assertTrue(iter.hasNext()); + Entry<Key,Value> entry = iter.next(); + + assertEquals(k, entry.getKey()); + assertEquals(v, entry.getValue()); + + } + } + + assertFalse(iter.hasNext()); + + for (int i = 0; i < 256; i++) { + for (int j = 0; j < 256; j++) { + byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)}; + byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)}; + + Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1); + Value v = new Value(data); + + Value v2 = nm.get(k); + + assertEquals(v, v2); + } + } + + nm.delete(); + } + + @Test + public void testEmpty() { + NativeMap nm = new NativeMap(); + + assertTrue(nm.size() == 0); + assertTrue(nm.getMemoryUsed() == 0); + + nm.delete(); + } + + @Test + public void testConcurrentIter() throws IOException { + NativeMap nm = new NativeMap(); + - nm.put(nk(0), nv(0)); - nm.put(nk(1), nv(1)); - nm.put(nk(3), nv(3)); ++ nm.put(newKey(0), newValue(0)); ++ nm.put(newKey(1), newValue(1)); ++ nm.put(newKey(3), newValue(3)); + + SortedKeyValueIterator<Key,Value> iter = nm.skvIterator(); + + // modify map after iter created - nm.put(nk(2), nv(2)); ++ nm.put(newKey(2), newValue(2)); + + assertTrue(iter.hasTop()); - assertEquals(iter.getTopKey(), nk(0)); ++ assertEquals(iter.getTopKey(), newKey(0)); + iter.next(); + + assertTrue(iter.hasTop()); - assertEquals(iter.getTopKey(), nk(1)); ++ assertEquals(iter.getTopKey(), newKey(1)); + iter.next(); + + assertTrue(iter.hasTop()); - assertEquals(iter.getTopKey(), nk(2)); ++ assertEquals(iter.getTopKey(), newKey(2)); + iter.next(); + + assertTrue(iter.hasTop()); - assertEquals(iter.getTopKey(), nk(3)); ++ assertEquals(iter.getTopKey(), newKey(3)); + iter.next(); + + assertFalse(iter.hasTop()); + + nm.delete(); + } + +}