This is an automated email from the ASF dual-hosted git repository. xiangfu pushed a commit to branch fixing_raw_bytes_comparison in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit bc76af5b63312984a67bee2e856843290d536b4e Author: Xiang Fu <fx19880...@gmail.com> AuthorDate: Wed Sep 4 15:34:47 2019 -0700 Adding more tests --- .../predicate/RangePredicateEvaluatorFactory.java | 16 ++++-- .../NoDictionaryEqualsPredicateEvaluatorsTest.java | 37 ++++++++++++++ .../NoDictionaryInPredicateEvaluatorTest.java | 40 +++++++++++++++ .../NoDictionaryRangePredicateEvaluatorTest.java | 57 ++++++++++++++++++++++ .../predicate/PredicateEvaluatorTestUtils.java | 7 +++ 5 files changed, 153 insertions(+), 4 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java index f2b7ad3..d9ab7e7 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java @@ -420,8 +420,16 @@ public class RangePredicateEvaluatorFactory { final boolean _includeUpperBoundary; BytesRawValueBasedRangePredicateEvaluator(RangePredicate rangePredicate) { - _lowerBoundary = BytesUtils.toBytes(rangePredicate.getLowerBoundary()); - _upperBoundary = BytesUtils.toBytes(rangePredicate.getUpperBoundary()); + if (!"*".equals(rangePredicate.getLowerBoundary())) { + _lowerBoundary = BytesUtils.toBytes(rangePredicate.getLowerBoundary()); + } else { + _lowerBoundary = null; + } + if (!"*".equals(rangePredicate.getUpperBoundary())) { + _upperBoundary = BytesUtils.toBytes(rangePredicate.getUpperBoundary()); + } else { + _upperBoundary = null; + } _includeLowerBoundary = rangePredicate.includeLowerBoundary(); _includeUpperBoundary = rangePredicate.includeUpperBoundary(); } @@ -434,14 +442,14 @@ public class RangePredicateEvaluatorFactory { @Override public boolean applySV(byte[] value) { boolean result = true; - if (!_lowerBoundary.equals("*")) { + if (_lowerBoundary != null) { if (_includeLowerBoundary) { result = ByteArray.compare(_lowerBoundary, value) <= 0; } else { result = ByteArray.compare(_lowerBoundary, value) < 0; } } - if (!_upperBoundary.equals("*")) { + if (_upperBoundary != null) { if (_includeUpperBoundary) { result &= ByteArray.compare(_upperBoundary, value) >= 0; } else { diff --git a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryEqualsPredicateEvaluatorsTest.java b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryEqualsPredicateEvaluatorsTest.java index 31b0220..823cdc3 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryEqualsPredicateEvaluatorsTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryEqualsPredicateEvaluatorsTest.java @@ -24,6 +24,7 @@ import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang3.mutable.MutableInt; import org.apache.pinot.common.data.FieldSpec; +import org.apache.pinot.common.utils.BytesUtils; import org.apache.pinot.core.common.predicate.EqPredicate; import org.apache.pinot.core.common.predicate.NEqPredicate; import org.apache.pinot.core.operator.filter.predicate.EqualsPredicateEvaluatorFactory; @@ -219,4 +220,40 @@ public class NoDictionaryEqualsPredicateEvaluatorsTest { !ArrayUtils.contains(randomStrings, stringValue)); } } + + @Test + public void testBytesPredicateEvaluators() { + byte[] bytesValue = RandomStringUtils.random(MAX_STRING_LENGTH).getBytes(); + String hexStringValue = BytesUtils.toHexString(bytesValue); + EqPredicate eqPredicate = new EqPredicate(COLUMN_NAME, Collections.singletonList(hexStringValue)); + PredicateEvaluator eqPredicateEvaluator = + EqualsPredicateEvaluatorFactory.newRawValueBasedEvaluator(eqPredicate, FieldSpec.DataType.BYTES); + + NEqPredicate neqPredicate = new NEqPredicate(COLUMN_NAME, Collections.singletonList(hexStringValue)); + PredicateEvaluator neqPredicateEvaluator = + NotEqualsPredicateEvaluatorFactory.newRawValueBasedEvaluator(neqPredicate, FieldSpec.DataType.BYTES); + + Assert.assertTrue(eqPredicateEvaluator.applySV(bytesValue)); + Assert.assertFalse(neqPredicateEvaluator.applySV(bytesValue)); + + byte[][] randomBytesArray = new byte[NUM_MULTI_VALUES][]; + PredicateEvaluatorTestUtils.fillRandom(randomBytesArray, MAX_STRING_LENGTH); + randomBytesArray[_random.nextInt(NUM_MULTI_VALUES)] = bytesValue; + + Assert.assertTrue(eqPredicateEvaluator.applyMV(randomBytesArray, NUM_MULTI_VALUES)); + Assert.assertFalse(neqPredicateEvaluator.applyMV(randomBytesArray, NUM_MULTI_VALUES)); + + for (int i = 0; i < 100; i++) { + byte[] randomBytes = RandomStringUtils.random(MAX_STRING_LENGTH).getBytes(); + String randomString = BytesUtils.toHexString(randomBytes); + Assert.assertEquals(eqPredicateEvaluator.applySV(randomBytes), (randomString.equals(hexStringValue))); + Assert.assertEquals(neqPredicateEvaluator.applySV(randomBytes), (!randomString.equals(hexStringValue))); + + PredicateEvaluatorTestUtils.fillRandom(randomBytesArray, MAX_STRING_LENGTH); + Assert.assertEquals(eqPredicateEvaluator.applyMV(randomBytesArray, NUM_MULTI_VALUES), + ArrayUtils.contains(randomBytesArray, hexStringValue)); + Assert.assertEquals(neqPredicateEvaluator.applyMV(randomBytesArray, NUM_MULTI_VALUES), + !ArrayUtils.contains(randomBytesArray, hexStringValue)); + } + } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryInPredicateEvaluatorTest.java b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryInPredicateEvaluatorTest.java index 217f186..9435aaf 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryInPredicateEvaluatorTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryInPredicateEvaluatorTest.java @@ -34,6 +34,7 @@ import java.util.Set; import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang3.mutable.MutableInt; import org.apache.pinot.common.data.FieldSpec; +import org.apache.pinot.common.utils.BytesUtils; import org.apache.pinot.core.common.predicate.InPredicate; import org.apache.pinot.core.common.predicate.NotInPredicate; import org.apache.pinot.core.operator.filter.predicate.InPredicateEvaluatorFactory; @@ -252,4 +253,43 @@ public class NoDictionaryInPredicateEvaluatorTest { Assert.assertTrue(inPredicateEvaluator.applyMV(multiValues, NUM_MULTI_VALUES)); Assert.assertFalse(notInPredicateEvaluator.applyMV(multiValues, NUM_MULTI_VALUES)); } + + @Test + public void testBytesPredicateEvaluators() { + List<String> stringValues = new ArrayList<>(NUM_PREDICATE_VALUES); + Set<byte[]> valueSet = new HashSet<>(); + + for (int i = 0; i < 100; i++) { + byte[] value = RandomStringUtils.random(MAX_STRING_LENGTH).getBytes(); + valueSet.add(value); + stringValues.add(BytesUtils.toHexString(value)); + } + + InPredicate inPredicate = new InPredicate(COLUMN_NAME, stringValues); + PredicateEvaluator inPredicateEvaluator = + InPredicateEvaluatorFactory.newRawValueBasedEvaluator(inPredicate, FieldSpec.DataType.BYTES); + + NotInPredicate notInPredicate = new NotInPredicate(COLUMN_NAME, stringValues); + PredicateEvaluator notInPredicateEvaluator = + NotInPredicateEvaluatorFactory.newRawValueBasedEvaluator(notInPredicate, FieldSpec.DataType.BYTES); + + for (byte[] value : valueSet) { + Assert.assertTrue(inPredicateEvaluator.applySV(value)); + Assert.assertFalse(notInPredicateEvaluator.applySV(value)); + } + + for (int i = 0; i < 100; i++) { + byte[] value = RandomStringUtils.random(MAX_STRING_LENGTH).getBytes(); + Assert.assertEquals(inPredicateEvaluator.applySV(value), valueSet.contains(value)); + Assert.assertEquals(notInPredicateEvaluator.applySV(value), !valueSet.contains(value)); + } + + byte[][] multiValues = new byte[NUM_MULTI_VALUES][]; + PredicateEvaluatorTestUtils.fillRandom(multiValues, MAX_STRING_LENGTH); + multiValues[_random.nextInt(NUM_MULTI_VALUES)] = + BytesUtils.toBytes(stringValues.get(_random.nextInt(NUM_PREDICATE_VALUES))); + + Assert.assertTrue(inPredicateEvaluator.applyMV(multiValues, NUM_MULTI_VALUES)); + Assert.assertFalse(notInPredicateEvaluator.applyMV(multiValues, NUM_MULTI_VALUES)); + } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryRangePredicateEvaluatorTest.java b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryRangePredicateEvaluatorTest.java index 5f68646..94887a2 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryRangePredicateEvaluatorTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryRangePredicateEvaluatorTest.java @@ -20,6 +20,8 @@ package org.apache.pinot.core.predicate; import java.util.Collections; import org.apache.pinot.common.data.FieldSpec; +import org.apache.pinot.common.utils.BytesUtils; +import org.apache.pinot.common.utils.primitive.ByteArray; import org.apache.pinot.core.common.predicate.RangePredicate; import org.apache.pinot.core.operator.filter.predicate.PredicateEvaluator; import org.apache.pinot.core.operator.filter.predicate.RangePredicateEvaluatorFactory; @@ -261,6 +263,61 @@ public class NoDictionaryRangePredicateEvaluatorTest { } } + @Test + public void testBytesPredicateEvaluator() { + + PredicateEvaluator predicateEvaluator = buildRangePredicate("[10\t\t20]", FieldSpec.DataType.BYTES); + for (int i = 0x00; i < 0x30; i++) { + byte[] value = Integer.toString(i).getBytes(); + Assert.assertEquals(predicateEvaluator.applySV(value), + (ByteArray.compare(value, new byte[]{0x10}) >= 0 && ByteArray.compare(value, new byte[]{0x20}) <= 0)); + } + + predicateEvaluator = buildRangePredicate("(10\t\t20]", FieldSpec.DataType.BYTES); + for (int i = 0x00; i < 0x30; i++) { + byte[] value = Integer.toString(i).getBytes(); + Assert.assertEquals(predicateEvaluator.applySV(value), + (ByteArray.compare(value, new byte[]{0x10}) > 0 && ByteArray.compare(value, new byte[]{0x20}) <= 0)); + } + + predicateEvaluator = buildRangePredicate("(10\t\t20)", FieldSpec.DataType.BYTES); + for (int i = 0x00; i < 0x30; i++) { + byte[] value = Integer.toString(i).getBytes(); + Assert.assertEquals(predicateEvaluator.applySV(value), + (ByteArray.compare(value, new byte[]{0x10}) > 0 && ByteArray.compare(value, new byte[]{0x20}) < 0)); + } + + predicateEvaluator = buildRangePredicate("(*\t\t10]", FieldSpec.DataType.BYTES); + for (int i = 0x00; i < 0x30; i++) { + byte[] value = Integer.toString(i).getBytes(); + Assert.assertEquals(predicateEvaluator.applySV(value), ByteArray.compare(value, new byte[]{0x10}) <= 0); + } + + predicateEvaluator = buildRangePredicate("(*\t\t10)", FieldSpec.DataType.BYTES); + for (int i = 0x00; i < 0x30; i++) { + byte[] value = Integer.toString(i).getBytes(); + Assert.assertEquals(predicateEvaluator.applySV(value), ByteArray.compare(value, new byte[]{0x10}) < 0); + } + + predicateEvaluator = buildRangePredicate("[10\t\t*)", FieldSpec.DataType.BYTES); + for (int i = 0x00; i < 0x30; i++) { + byte[] value = Integer.toString(i).getBytes(); + Assert.assertEquals(predicateEvaluator.applySV(value), ByteArray.compare(value, new byte[]{0x10}) >= 0); + } + + predicateEvaluator = buildRangePredicate("(10\t\t*)", FieldSpec.DataType.BYTES); + for (int i = 0x00; i < 0x30; i++) { + byte[] value = Integer.toString(i).getBytes(); + Assert.assertEquals(predicateEvaluator.applySV(value), ByteArray.compare(value, new byte[]{0x10}) > 0); + } + + predicateEvaluator = buildRangePredicate("(*\t\t*)", FieldSpec.DataType.BYTES); + for (int i = 0x00; i < 0x30; i++) { + byte[] value = Integer.toString(i).getBytes(); + Assert.assertTrue(predicateEvaluator.applySV(value)); + } + } + private PredicateEvaluator buildRangePredicate(String rangeString, FieldSpec.DataType dataType) { RangePredicate predicate = new RangePredicate(COLUMN_NAME, Collections.singletonList(rangeString)); return RangePredicateEvaluatorFactory.newRawValueBasedEvaluator(predicate, dataType); diff --git a/pinot-core/src/test/java/org/apache/pinot/core/predicate/PredicateEvaluatorTestUtils.java b/pinot-core/src/test/java/org/apache/pinot/core/predicate/PredicateEvaluatorTestUtils.java index 6af6e44..7f33f09 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/predicate/PredicateEvaluatorTestUtils.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/predicate/PredicateEvaluatorTestUtils.java @@ -20,6 +20,7 @@ package org.apache.pinot.core.predicate; import java.util.Random; import org.apache.commons.lang.RandomStringUtils; +import org.apache.pinot.common.utils.BytesUtils; public class PredicateEvaluatorTestUtils { @@ -54,4 +55,10 @@ public class PredicateEvaluatorTestUtils { randomValues[i] = RandomStringUtils.random(maxStringLength); } } + + public static void fillRandom(byte[][] randomValues, int maxStringLength) { + for (int i = 0; i < randomValues.length; i++) { + randomValues[i] = RandomStringUtils.random(maxStringLength).getBytes(); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org