ACCUMULO-626 Slight cleanup to Dylan's changes. Also fixes various build-related issues.
Closes joshelser/accumulo#2 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c7b27e0a Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c7b27e0a Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c7b27e0a Branch: refs/heads/master Commit: c7b27e0ae42f1046243126abf2853356b6300023 Parents: 822bc5a Author: Josh Elser <[email protected]> Authored: Fri Dec 11 00:30:21 2015 -0500 Committer: Josh Elser <[email protected]> Committed: Mon Dec 14 00:29:34 2015 -0500 ---------------------------------------------------------------------- .../src/main/findbugs/exclude-filter.xml | 18 +++ .../iteratortest/IteratorTestOutput.java | 11 ++ .../junit4/BaseJUnit4IteratorTest.java | 4 +- .../testcases/DeepCopyTestCase.java | 3 +- .../testcases/IsolatedDeepCopiesTestCase.java | 14 +- .../testcases/MultipleHasTopCalls.java | 3 +- .../iteratortest/testcases/ReSeekTestCase.java | 11 +- .../iteratortest/WholeRowIteratorTest.java | 147 ------------------ pom.xml | 5 + test/pom.xml | 5 + .../test/iterator/AgeOffFilterTest.java | 138 +++++++++++++++++ .../test/iterator/CfCqSliceFilterTest.java | 132 ++++++++++++++++ .../test/iterator/SummingCombinerTest.java | 132 ++++++++++++++++ .../test/iterator/WholeRowIteratorTest.java | 150 +++++++++++++++++++ 14 files changed, 613 insertions(+), 160 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b27e0a/iterator-test-harness/src/main/findbugs/exclude-filter.xml ---------------------------------------------------------------------- diff --git a/iterator-test-harness/src/main/findbugs/exclude-filter.xml b/iterator-test-harness/src/main/findbugs/exclude-filter.xml new file mode 100644 index 0000000..c801230 --- /dev/null +++ b/iterator-test-harness/src/main/findbugs/exclude-filter.xml @@ -0,0 +1,18 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<FindBugsFilter> +</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b27e0a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestOutput.java ---------------------------------------------------------------------- diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestOutput.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestOutput.java index 0a19727..806c428 100644 --- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestOutput.java +++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestOutput.java @@ -119,6 +119,16 @@ public class IteratorTestOutput { } @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((exception == null) ? 0 : exception.hashCode()); + result = prime * result + ((outcome == null) ? 0 : outcome.hashCode()); + result = prime * result + ((output == null) ? 0 : output.hashCode()); + return result; + } + + @Override public boolean equals(Object o) { if (!(o instanceof IteratorTestOutput)) { return false; @@ -141,6 +151,7 @@ public class IteratorTestOutput { return false; } return exception.equals(other.getException()); + } @Override http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b27e0a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/junit4/BaseJUnit4IteratorTest.java ---------------------------------------------------------------------- diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/junit4/BaseJUnit4IteratorTest.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/junit4/BaseJUnit4IteratorTest.java index 66e9dbc..6325ae6 100644 --- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/junit4/BaseJUnit4IteratorTest.java +++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/junit4/BaseJUnit4IteratorTest.java @@ -40,7 +40,7 @@ import org.slf4j.LoggerFactory; * A base JUnit4 test class for users to leverage with the JUnit Parameterized Runner. * <p> * Users should extend this class and implement a static method using the {@code @Parameters} annotation. - * + * * <pre> * @Parameters * public static Object[][] data() { @@ -50,7 +50,7 @@ import org.slf4j.LoggerFactory; * return BaseJUnit4IteratorTest.createParameters(input, expectedOutput, testCases); * } * </pre> - * + * */ @RunWith(Parameterized.class) public class BaseJUnit4IteratorTest { http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b27e0a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/DeepCopyTestCase.java ---------------------------------------------------------------------- diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/DeepCopyTestCase.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/DeepCopyTestCase.java index 1a608c1..3c3e6da 100644 --- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/DeepCopyTestCase.java +++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/DeepCopyTestCase.java @@ -54,7 +54,8 @@ public class DeepCopyTestCase extends OutputVerifyingTestCase { TreeMap<Key,Value> consume(SortedKeyValueIterator<Key,Value> skvi) throws IOException { TreeMap<Key,Value> data = new TreeMap<>(); while (skvi.hasTop()) { - data.put(skvi.getTopKey(), skvi.getTopValue()); + // Make sure to copy the K-V + data.put(new Key(skvi.getTopKey()), new Value(skvi.getTopValue())); skvi.next(); } return data; http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b27e0a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/IsolatedDeepCopiesTestCase.java ---------------------------------------------------------------------- diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/IsolatedDeepCopiesTestCase.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/IsolatedDeepCopiesTestCase.java index 2bc1e63..1b8b05f 100644 --- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/IsolatedDeepCopiesTestCase.java +++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/IsolatedDeepCopiesTestCase.java @@ -17,6 +17,7 @@ package org.apache.accumulo.iteratortest.testcases; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -33,11 +34,14 @@ import org.apache.accumulo.iteratortest.IteratorTestInput; import org.apache.accumulo.iteratortest.IteratorTestOutput; import org.apache.accumulo.iteratortest.IteratorTestUtil; import org.apache.accumulo.iteratortest.environments.SimpleIteratorEnvironment; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Test case that verifies that copies do not impact one another. */ public class IsolatedDeepCopiesTestCase extends OutputVerifyingTestCase { + private static final Logger log = LoggerFactory.getLogger(IsolatedDeepCopiesTestCase.class); private final Random random = new Random(); @@ -60,7 +64,7 @@ public class IsolatedDeepCopiesTestCase extends OutputVerifyingTestCase { copy1.seek(testInput.getRange(), seekColumnFamilies, seekInclusive); copy2.seek(testInput.getRange(), seekColumnFamilies, seekInclusive); - TreeMap<Key,Value> output = consumeMany(Arrays.asList(skvi, copy1, copy2), seekRange, seekColumnFamilies, seekInclusive); + TreeMap<Key,Value> output = consumeMany(new ArrayList<>(Arrays.asList(skvi, copy1, copy2)), seekRange, seekColumnFamilies, seekInclusive); return new IteratorTestOutput(output); } catch (IOException e) { @@ -75,8 +79,10 @@ public class IsolatedDeepCopiesTestCase extends OutputVerifyingTestCase { while (allHasTop(iterators)) { // occasionally deep copy one of the existing iterators if (random.nextInt(3) == 0) { + log.debug("Deep-copying and re-seeking an iterator"); SortedKeyValueIterator<Key,Value> newcopy = getRandomElement(iterators).deepCopy(new SimpleIteratorEnvironment()); newcopy.seek(new Range(getTopKey(iterators), true, range.getEndKey(), range.isEndKeyInclusive()), seekColumnFamilies, seekInclusive); + // keep using the new one too, should act like the others iterators.add(newcopy); } @@ -126,7 +132,8 @@ public class IsolatedDeepCopiesTestCase extends OutputVerifyingTestCase { } } - return topKey; + // Copy the key + return new Key(topKey); } Value getTopValue(Collection<SortedKeyValueIterator<Key,Value>> iterators) { @@ -141,7 +148,8 @@ public class IsolatedDeepCopiesTestCase extends OutputVerifyingTestCase { } } - return topValue; + // Copy the value + return new Value(topValue); } void next(Collection<SortedKeyValueIterator<Key,Value>> iterators) throws IOException { http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b27e0a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/MultipleHasTopCalls.java ---------------------------------------------------------------------- diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/MultipleHasTopCalls.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/MultipleHasTopCalls.java index 34bf776..087516d 100644 --- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/MultipleHasTopCalls.java +++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/MultipleHasTopCalls.java @@ -67,7 +67,8 @@ public class MultipleHasTopCalls extends OutputVerifyingTestCase { throw badStateException(true); } } - data.put(skvi.getTopKey(), skvi.getTopValue()); + // Make sure to copy the K-V + data.put(new Key(skvi.getTopKey()), new Value(skvi.getTopValue())); skvi.next(); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b27e0a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/ReSeekTestCase.java ---------------------------------------------------------------------- diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/ReSeekTestCase.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/ReSeekTestCase.java index d539526..512202c 100644 --- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/ReSeekTestCase.java +++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/ReSeekTestCase.java @@ -71,14 +71,13 @@ public class ReSeekTestCase extends OutputVerifyingTestCase { int i = 0; while (skvi.hasTop()) { - data.put(skvi.getTopKey(), skvi.getTopValue()); + data.put(new Key(skvi.getTopKey()), new Value(skvi.getTopValue())); /* - * One of the trickiest cases in writing iterators: - * - * After any result is returned from a TabletServer to the client, the Iterator in the TabletServer's memory may be torn down. To preserve the state and - * guarantee that all records are received, the TabletServer does remember the last Key it returned to the client. It will recreate the Iterator (stack), - * and seek it using an updated Range. This range's start key is set to the last Key returned, non-inclusive. + * One of the trickiest cases in writing iterators: After any result is returned from a TabletServer to the client, the Iterator in the TabletServer's + * memory may be torn down. To preserve the state and guarantee that all records are received, the TabletServer does remember the last Key it returned to + * the client. It will recreate the Iterator (stack), and seek it using an updated Range. This range's start key is set to the last Key returned, + * non-inclusive. */ if (i % RESEEK_INTERVAL == reseekCount) { // Last key http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b27e0a/iterator-test-harness/src/test/java/org/apache/accumulo/iteratortest/WholeRowIteratorTest.java ---------------------------------------------------------------------- diff --git a/iterator-test-harness/src/test/java/org/apache/accumulo/iteratortest/WholeRowIteratorTest.java b/iterator-test-harness/src/test/java/org/apache/accumulo/iteratortest/WholeRowIteratorTest.java deleted file mode 100644 index 0b116f2..0000000 --- a/iterator-test-harness/src/test/java/org/apache/accumulo/iteratortest/WholeRowIteratorTest.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.iteratortest; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map.Entry; -import java.util.TreeMap; - -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.user.WholeRowIterator; -import org.apache.accumulo.iteratortest.junit4.BaseJUnit4IteratorTest; -import org.apache.accumulo.iteratortest.testcases.IteratorTestCase; -import org.apache.hadoop.io.Text; -import org.junit.runners.Parameterized.Parameters; - -/** - * Framework tests for {@link WholeRowIterator}. - */ -public class WholeRowIteratorTest extends BaseJUnit4IteratorTest { - - @Parameters - public static Object[][] parameters() { - IteratorTestInput input = getIteratorInput(); - IteratorTestOutput output = getIteratorOutput(); - List<IteratorTestCase> tests = IteratorTestCaseFinder.findAllTestCases(); - return BaseJUnit4IteratorTest.createParameters(input, output, tests); - } - - private static final TreeMap<Key,Value> INPUT_DATA = createInputData(); - private static final TreeMap<Key,Value> OUTPUT_DATA = createOutputData(); - - private static TreeMap<Key,Value> createInputData() { - TreeMap<Key,Value> data = new TreeMap<>(); - - data.put(new Key("1", "", "a"), new Value("1a".getBytes())); - data.put(new Key("1", "", "b"), new Value("1b".getBytes())); - data.put(new Key("1", "a", "a"), new Value("1aa".getBytes())); - data.put(new Key("1", "a", "b"), new Value("1ab".getBytes())); - data.put(new Key("1", "b", "a"), new Value("1ba".getBytes())); - - data.put(new Key("2", "a", "a"), new Value("2aa".getBytes())); - data.put(new Key("2", "a", "b"), new Value("2ab".getBytes())); - data.put(new Key("2", "a", "c"), new Value("2ac".getBytes())); - data.put(new Key("2", "c", "c"), new Value("2cc".getBytes())); - - data.put(new Key("3", "a", ""), new Value("3a".getBytes())); - - data.put(new Key("4", "a", "b"), new Value("4ab".getBytes())); - - data.put(new Key("5", "a", "a"), new Value("5aa".getBytes())); - data.put(new Key("5", "a", "b"), new Value("5ab".getBytes())); - data.put(new Key("5", "a", "c"), new Value("5ac".getBytes())); - data.put(new Key("5", "a", "d"), new Value("5ad".getBytes())); - - data.put(new Key("6", "", "a"), new Value("6a".getBytes())); - data.put(new Key("6", "", "b"), new Value("6b".getBytes())); - data.put(new Key("6", "", "c"), new Value("6c".getBytes())); - data.put(new Key("6", "", "d"), new Value("6d".getBytes())); - data.put(new Key("6", "", "e"), new Value("6e".getBytes())); - data.put(new Key("6", "1", "a"), new Value("61a".getBytes())); - data.put(new Key("6", "1", "b"), new Value("61b".getBytes())); - data.put(new Key("6", "1", "c"), new Value("61c".getBytes())); - data.put(new Key("6", "1", "d"), new Value("61d".getBytes())); - data.put(new Key("6", "1", "e"), new Value("61e".getBytes())); - - return data; - } - - private static TreeMap<Key,Value> createOutputData() { - TreeMap<Key,Value> data = new TreeMap<>(); - - Text row = null; - List<Key> keys = new ArrayList<>(); - List<Value> values = new ArrayList<>(); - - // Generate the output data from the input data - for (Entry<Key,Value> entry : INPUT_DATA.entrySet()) { - if (null == row) { - row = entry.getKey().getRow(); - } - - if (!row.equals(entry.getKey().getRow())) { - // Moved to the next row - try { - // Serialize and save - Value encoded = WholeRowIterator.encodeRow(keys, values); - data.put(new Key(row), encoded); - } catch (IOException e) { - throw new RuntimeException(e); - } - - // Empty the aggregated k-v's - keys = new ArrayList<>(); - values = new ArrayList<>(); - // Set the new current row - row = entry.getKey().getRow(); - } - - // Aggregate the current row - keys.add(entry.getKey()); - values.add(entry.getValue()); - } - - if (!keys.isEmpty()) { - try { - Value encoded = WholeRowIterator.encodeRow(keys, values); - data.put(new Key(row), encoded); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - return data; - } - - private static IteratorTestInput getIteratorInput() { - return new IteratorTestInput(WholeRowIterator.class, Collections.<String,String> emptyMap(), new Range(), INPUT_DATA); - } - - private static IteratorTestOutput getIteratorOutput() { - return new IteratorTestOutput(OUTPUT_DATA); - } - - public WholeRowIteratorTest(IteratorTestInput input, IteratorTestOutput expectedOutput, IteratorTestCase testCase) { - super(input, expectedOutput, testCase); - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b27e0a/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index c18342e..d16e90c 100644 --- a/pom.xml +++ b/pom.xml @@ -276,6 +276,11 @@ </dependency> <dependency> <groupId>org.apache.accumulo</groupId> + <artifactId>accumulo-iterator-test-harness</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-master</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b27e0a/test/pom.xml ---------------------------------------------------------------------- diff --git a/test/pom.xml b/test/pom.xml index 01b27da..f2ccfaf 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -186,6 +186,11 @@ <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> </dependency> + <dependency> + <groupId>org.apache.accumulo</groupId> + <artifactId>accumulo-iterator-test-harness</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <pluginManagement> http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b27e0a/test/src/test/java/org/apache/accumulo/test/iterator/AgeOffFilterTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/iterator/AgeOffFilterTest.java b/test/src/test/java/org/apache/accumulo/test/iterator/AgeOffFilterTest.java new file mode 100644 index 0000000..e78d8a9 --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/iterator/AgeOffFilterTest.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.iterator; + +import static org.junit.Assert.assertNotNull; + +import java.util.List; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.user.AgeOffFilter; +import org.apache.accumulo.iteratortest.IteratorTestCaseFinder; +import org.apache.accumulo.iteratortest.IteratorTestInput; +import org.apache.accumulo.iteratortest.IteratorTestOutput; +import org.apache.accumulo.iteratortest.junit4.BaseJUnit4IteratorTest; +import org.apache.accumulo.iteratortest.testcases.IteratorTestCase; +import org.junit.runners.Parameterized.Parameters; + +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; + +/** + * Iterator test harness tests for AgeOffFilter + */ +public class AgeOffFilterTest extends BaseJUnit4IteratorTest { + public static long NOW; + public static long TTL; + + @Parameters + public static Object[][] parameters() { + // Test ageoff after 30 seconds. + NOW = System.currentTimeMillis(); + TTL = 30 * 1000; + + IteratorTestInput input = getIteratorInput(); + IteratorTestOutput output = getIteratorOutput(); + List<IteratorTestCase> tests = IteratorTestCaseFinder.findAllTestCases(); + return BaseJUnit4IteratorTest.createParameters(input, output, tests); + } + + private static final TreeMap<Key,Value> INPUT_DATA = createInputData(); + private static final TreeMap<Key,Value> OUTPUT_DATA = createOutputData(); + + private static TreeMap<Key,Value> createInputData() { + TreeMap<Key,Value> data = new TreeMap<>(); + final Value value = new Value(new byte[] {'a'}); + + data.put(new Key("1", "a", "a", nowDelta(25)), value); + data.put(new Key("2", "a", "a", nowDelta(35)), value); + data.put(new Key("3", "a", "a", nowDelta(55)), value); + data.put(new Key("4", "a", "a", nowDelta(0)), value); + data.put(new Key("5", "a", "a", nowDelta(-29)), value); + data.put(new Key("6", "a", "a", nowDelta(-28)), value); + // Dropped + data.put(new Key("7", "a", "a", nowDelta(-40)), value); + // Dropped (comparison is not inclusive) + data.put(new Key("8", "a", "a", nowDelta(-30)), value); + // Dropped + data.put(new Key("9", "a", "a", nowDelta(-31)), value); + + // Dropped + data.put(new Key("a", "", "", nowDelta(-50)), value); + data.put(new Key("a", "a", "", nowDelta(-20)), value); + data.put(new Key("a", "a", "a", nowDelta(50)), value); + data.put(new Key("a", "a", "b", nowDelta(-15)), value); + // Dropped + data.put(new Key("a", "a", "c", nowDelta(-32)), value); + // Dropped + data.put(new Key("a", "a", "d", nowDelta(-32)), value); + + return data; + } + + /** + * Compute a timestamp (milliseconds) based on {@link #NOW} plus the <code>seconds</code> argument. + * + * @param seconds + * The number of seconds to add to <code>NOW</code> . + * @return A Key timestamp the provided number of seconds after <code>NOW</code>. + */ + private static long nowDelta(long seconds) { + return NOW + (seconds * 1000); + } + + private static TreeMap<Key,Value> createOutputData() { + TreeMap<Key,Value> data = new TreeMap<>(); + + Iterable<Entry<Key,Value>> filtered = Iterables.filter(data.entrySet(), new Predicate<Entry<Key,Value>>() { + + @Override + public boolean apply(Entry<Key,Value> input) { + assertNotNull(input); + return NOW - input.getKey().getTimestamp() > TTL; + } + + }); + + for (Entry<Key,Value> entry : filtered) { + data.put(entry.getKey(), entry.getValue()); + } + + return data; + } + + private static IteratorTestInput getIteratorInput() { + IteratorSetting setting = new IteratorSetting(50, AgeOffFilter.class); + AgeOffFilter.setCurrentTime(setting, NOW); + AgeOffFilter.setTTL(setting, TTL); + return new IteratorTestInput(AgeOffFilter.class, setting.getOptions(), new Range(), INPUT_DATA); + } + + private static IteratorTestOutput getIteratorOutput() { + return new IteratorTestOutput(OUTPUT_DATA); + } + + public AgeOffFilterTest(IteratorTestInput input, IteratorTestOutput expectedOutput, IteratorTestCase testCase) { + super(input, expectedOutput, testCase); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b27e0a/test/src/test/java/org/apache/accumulo/test/iterator/CfCqSliceFilterTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/iterator/CfCqSliceFilterTest.java b/test/src/test/java/org/apache/accumulo/test/iterator/CfCqSliceFilterTest.java new file mode 100644 index 0000000..fc0f672 --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/iterator/CfCqSliceFilterTest.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.iterator; + +import static org.junit.Assert.assertNotNull; + +import java.util.HashMap; +import java.util.List; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.user.CfCqSliceFilter; +import org.apache.accumulo.core.iterators.user.CfCqSliceOpts; +import org.apache.accumulo.iteratortest.IteratorTestCaseFinder; +import org.apache.accumulo.iteratortest.IteratorTestInput; +import org.apache.accumulo.iteratortest.IteratorTestOutput; +import org.apache.accumulo.iteratortest.junit4.BaseJUnit4IteratorTest; +import org.apache.accumulo.iteratortest.testcases.IteratorTestCase; +import org.junit.runners.Parameterized.Parameters; + +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; + +/** + * Iterator test harness tests for CfCqSliceFilter + */ +public class CfCqSliceFilterTest extends BaseJUnit4IteratorTest { + + // Default is inclusive on min and max + public static final String MIN_CF = "f"; + public static final String MAX_CF = "m"; + public static final String MIN_CQ = "q"; + public static final String MAX_CQ = "y"; + + @Parameters + public static Object[][] parameters() { + IteratorTestInput input = getIteratorInput(); + IteratorTestOutput output = getIteratorOutput(); + List<IteratorTestCase> tests = IteratorTestCaseFinder.findAllTestCases(); + return BaseJUnit4IteratorTest.createParameters(input, output, tests); + } + + private static final TreeMap<Key,Value> INPUT_DATA = createInputData(); + private static final TreeMap<Key,Value> OUTPUT_DATA = createOutputData(); + + private static TreeMap<Key,Value> createInputData() { + TreeMap<Key,Value> data = new TreeMap<>(); + Value value = new Value(new byte[] {'a'}); + + // Dropped + data.put(new Key("1", "a", "g"), value); + data.put(new Key("1", "f", "q"), value); + data.put(new Key("1", "f", "t"), value); + data.put(new Key("1", "g", "q"), value); + data.put(new Key("1", "g", "y"), value); + // Dropped + data.put(new Key("1", "g", "z"), value); + + // Dropped + data.put(new Key("2", "m", "a"), value); + + data.put(new Key("3", "j", "u"), value); + + data.put(new Key("4", "h", "w"), value); + data.put(new Key("4", "h", "x"), value); + data.put(new Key("4", "h", "y"), value); + data.put(new Key("4", "l", "r"), value); + // Dropped + data.put(new Key("4", "l", "z"), value); + data.put(new Key("4", "m", "y"), value); + + return data; + } + + private static TreeMap<Key,Value> createOutputData() { + TreeMap<Key,Value> data = new TreeMap<>(); + + Iterable<Entry<Key,Value>> filtered = Iterables.filter(INPUT_DATA.entrySet(), new Predicate<Entry<Key,Value>>() { + + @Override + public boolean apply(Entry<Key,Value> entry) { + assertNotNull(entry); + String cf = entry.getKey().getColumnFamily().toString(); + String cq = entry.getKey().getColumnQualifier().toString(); + return MIN_CF.compareTo(cf) <= 0 && MAX_CF.compareTo(cf) >= 0 && MIN_CQ.compareTo(cq) <= 0 && MAX_CQ.compareTo(cq) >= 0; + } + + }); + + for (Entry<Key,Value> entry : filtered) { + data.put(entry.getKey(), entry.getValue()); + } + + return data; + } + + private static IteratorTestInput getIteratorInput() { + HashMap<String,String> options = new HashMap<>(); + options.put(CfCqSliceOpts.OPT_MIN_CF, MIN_CF); + options.put(CfCqSliceOpts.OPT_MAX_CF, MAX_CF); + options.put(CfCqSliceOpts.OPT_MIN_CQ, MIN_CQ); + options.put(CfCqSliceOpts.OPT_MAX_CQ, MAX_CQ); + + return new IteratorTestInput(CfCqSliceFilter.class, options, new Range(), INPUT_DATA); + } + + private static IteratorTestOutput getIteratorOutput() { + return new IteratorTestOutput(OUTPUT_DATA); + } + + public CfCqSliceFilterTest(IteratorTestInput input, IteratorTestOutput expectedOutput, IteratorTestCase testCase) { + super(input, expectedOutput, testCase); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b27e0a/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java b/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java new file mode 100644 index 0000000..7bd06fe --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.iterator; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.util.List; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.TreeMap; + +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.PartialKey; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.Combiner; +import org.apache.accumulo.core.iterators.LongCombiner; +import org.apache.accumulo.core.iterators.user.SummingCombiner; +import org.apache.accumulo.iteratortest.IteratorTestCaseFinder; +import org.apache.accumulo.iteratortest.IteratorTestInput; +import org.apache.accumulo.iteratortest.IteratorTestOutput; +import org.apache.accumulo.iteratortest.junit4.BaseJUnit4IteratorTest; +import org.apache.accumulo.iteratortest.testcases.IteratorTestCase; +import org.junit.runners.Parameterized.Parameters; + +/** + * Iterator test harness tests for SummingCombiner + */ +public class SummingCombinerTest extends BaseJUnit4IteratorTest { + + @Parameters + public static Object[][] parameters() { + IteratorTestInput input = getIteratorInput(); + IteratorTestOutput output = getIteratorOutput(); + List<IteratorTestCase> tests = IteratorTestCaseFinder.findAllTestCases(); + return BaseJUnit4IteratorTest.createParameters(input, output, tests); + } + + private static final TreeMap<Key,Value> INPUT_DATA = createInputData(); + private static final TreeMap<Key,Value> OUTPUT_DATA = createOutputData(); + + private static TreeMap<Key,Value> createInputData() { + TreeMap<Key,Value> data = new TreeMap<>(); + + // 3 + data.put(new Key("1", "a", "a", 1), new Value(bytes("1"))); + data.put(new Key("1", "a", "a", 5), new Value(bytes("1"))); + data.put(new Key("1", "a", "a", 10), new Value(bytes("1"))); + // 7 + data.put(new Key("1", "a", "b", 1), new Value(bytes("5"))); + data.put(new Key("1", "a", "b", 5), new Value(bytes("2"))); + // 0 + data.put(new Key("1", "a", "f", 1), new Value(bytes("0"))); + // -10 + data.put(new Key("1", "a", "g", 5), new Value(bytes("1"))); + data.put(new Key("1", "a", "g", 10), new Value(bytes("-11"))); + // -5 + data.put(new Key("1", "b", "d", 10), new Value(bytes("-5"))); + // MAX_VALUE + data.put(new Key("1", "b", "e", 10), new Value(bytes(Long.toString(Long.MAX_VALUE)))); + // MIN_VALUE + data.put(new Key("1", "d", "d", 10), new Value(bytes(Long.toString(Long.MIN_VALUE)))); + // 30 + data.put(new Key("2", "a", "a", 1), new Value(bytes("5"))); + data.put(new Key("2", "a", "a", 5), new Value(bytes("10"))); + data.put(new Key("2", "a", "a", 10), new Value(bytes("15"))); + + return data; + } + + private static final byte[] bytes(String value) { + return Objects.requireNonNull(value).getBytes(UTF_8); + } + + private static TreeMap<Key,Value> createOutputData() { + TreeMap<Key,Value> data = new TreeMap<>(); + + Key lastKey = null; + long sum = 0; + for (Entry<Key,Value> entry : INPUT_DATA.entrySet()) { + if (null == lastKey) { + lastKey = entry.getKey(); + sum += Long.parseLong(entry.getValue().toString()); + } else { + if (0 != lastKey.compareTo(entry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS)) { + // Different key, store the running sum. + data.put(lastKey, new Value(Long.toString(sum).getBytes(UTF_8))); + // Reset lastKey and the sum + lastKey = entry.getKey(); + sum = 0; + } + + sum += Long.parseLong(entry.getValue().toString()); + } + } + + data.put(lastKey, new Value(Long.toString(sum).getBytes(UTF_8))); + + return data; + } + + private static IteratorTestInput getIteratorInput() { + IteratorSetting setting = new IteratorSetting(50, SummingCombiner.class); + LongCombiner.setEncodingType(setting, LongCombiner.Type.STRING); + Combiner.setCombineAllColumns(setting, true); + return new IteratorTestInput(SummingCombiner.class, setting.getOptions(), new Range(), INPUT_DATA); + } + + private static IteratorTestOutput getIteratorOutput() { + return new IteratorTestOutput(OUTPUT_DATA); + } + + public SummingCombinerTest(IteratorTestInput input, IteratorTestOutput expectedOutput, IteratorTestCase testCase) { + super(input, expectedOutput, testCase); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b27e0a/test/src/test/java/org/apache/accumulo/test/iterator/WholeRowIteratorTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/iterator/WholeRowIteratorTest.java b/test/src/test/java/org/apache/accumulo/test/iterator/WholeRowIteratorTest.java new file mode 100644 index 0000000..5e26106 --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/iterator/WholeRowIteratorTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.iterator; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.user.WholeRowIterator; +import org.apache.accumulo.iteratortest.IteratorTestCaseFinder; +import org.apache.accumulo.iteratortest.IteratorTestInput; +import org.apache.accumulo.iteratortest.IteratorTestOutput; +import org.apache.accumulo.iteratortest.junit4.BaseJUnit4IteratorTest; +import org.apache.accumulo.iteratortest.testcases.IteratorTestCase; +import org.apache.hadoop.io.Text; +import org.junit.runners.Parameterized.Parameters; + +/** + * Framework tests for {@link WholeRowIterator}. + */ +public class WholeRowIteratorTest extends BaseJUnit4IteratorTest { + + @Parameters + public static Object[][] parameters() { + IteratorTestInput input = getIteratorInput(); + IteratorTestOutput output = getIteratorOutput(); + List<IteratorTestCase> tests = IteratorTestCaseFinder.findAllTestCases(); + return BaseJUnit4IteratorTest.createParameters(input, output, tests); + } + + private static final TreeMap<Key,Value> INPUT_DATA = createInputData(); + private static final TreeMap<Key,Value> OUTPUT_DATA = createOutputData(); + + private static TreeMap<Key,Value> createInputData() { + TreeMap<Key,Value> data = new TreeMap<>(); + + data.put(new Key("1", "", "a"), new Value("1a".getBytes())); + data.put(new Key("1", "", "b"), new Value("1b".getBytes())); + data.put(new Key("1", "a", "a"), new Value("1aa".getBytes())); + data.put(new Key("1", "a", "b"), new Value("1ab".getBytes())); + data.put(new Key("1", "b", "a"), new Value("1ba".getBytes())); + + data.put(new Key("2", "a", "a"), new Value("2aa".getBytes())); + data.put(new Key("2", "a", "b"), new Value("2ab".getBytes())); + data.put(new Key("2", "a", "c"), new Value("2ac".getBytes())); + data.put(new Key("2", "c", "c"), new Value("2cc".getBytes())); + + data.put(new Key("3", "a", ""), new Value("3a".getBytes())); + + data.put(new Key("4", "a", "b"), new Value("4ab".getBytes())); + + data.put(new Key("5", "a", "a"), new Value("5aa".getBytes())); + data.put(new Key("5", "a", "b"), new Value("5ab".getBytes())); + data.put(new Key("5", "a", "c"), new Value("5ac".getBytes())); + data.put(new Key("5", "a", "d"), new Value("5ad".getBytes())); + + data.put(new Key("6", "", "a"), new Value("6a".getBytes())); + data.put(new Key("6", "", "b"), new Value("6b".getBytes())); + data.put(new Key("6", "", "c"), new Value("6c".getBytes())); + data.put(new Key("6", "", "d"), new Value("6d".getBytes())); + data.put(new Key("6", "", "e"), new Value("6e".getBytes())); + data.put(new Key("6", "1", "a"), new Value("61a".getBytes())); + data.put(new Key("6", "1", "b"), new Value("61b".getBytes())); + data.put(new Key("6", "1", "c"), new Value("61c".getBytes())); + data.put(new Key("6", "1", "d"), new Value("61d".getBytes())); + data.put(new Key("6", "1", "e"), new Value("61e".getBytes())); + + return data; + } + + private static TreeMap<Key,Value> createOutputData() { + TreeMap<Key,Value> data = new TreeMap<>(); + + Text row = null; + List<Key> keys = new ArrayList<>(); + List<Value> values = new ArrayList<>(); + + // Generate the output data from the input data + for (Entry<Key,Value> entry : INPUT_DATA.entrySet()) { + if (null == row) { + row = entry.getKey().getRow(); + } + + if (!row.equals(entry.getKey().getRow())) { + // Moved to the next row + try { + // Serialize and save + Value encoded = WholeRowIterator.encodeRow(keys, values); + data.put(new Key(row), encoded); + } catch (IOException e) { + throw new RuntimeException(e); + } + + // Empty the aggregated k-v's + keys = new ArrayList<>(); + values = new ArrayList<>(); + // Set the new current row + row = entry.getKey().getRow(); + } + + // Aggregate the current row + keys.add(entry.getKey()); + values.add(entry.getValue()); + } + + if (!keys.isEmpty()) { + try { + Value encoded = WholeRowIterator.encodeRow(keys, values); + data.put(new Key(row), encoded); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + return data; + } + + private static IteratorTestInput getIteratorInput() { + return new IteratorTestInput(WholeRowIterator.class, Collections.<String,String> emptyMap(), new Range(), INPUT_DATA); + } + + private static IteratorTestOutput getIteratorOutput() { + return new IteratorTestOutput(OUTPUT_DATA); + } + + public WholeRowIteratorTest(IteratorTestInput input, IteratorTestOutput expectedOutput, IteratorTestCase testCase) { + super(input, expectedOutput, testCase); + } + +}
