This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch fixing_raw_bytes_comparison
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/fixing_raw_bytes_comparison by
this push:
new 81438f1 Adding more tests
81438f1 is described below
commit 81438f14684333a7b83ab8244ab87b09613ce8ba
Author: Xiang Fu <[email protected]>
AuthorDate: Wed Sep 4 15:34:47 2019 -0700
Adding more tests
---
.../predicate/RangePredicateEvaluatorFactory.java | 16 ++++--
.../NoDictionaryEqualsPredicateEvaluatorsTest.java | 37 ++++++++++++++
.../NoDictionaryInPredicateEvaluatorTest.java | 40 +++++++++++++++
.../NoDictionaryRangePredicateEvaluatorTest.java | 57 ++++++++++++++++++++++
.../predicate/PredicateEvaluatorTestUtils.java | 7 +++
5 files changed, 153 insertions(+), 4 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
index f2b7ad3..d9ab7e7 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
@@ -420,8 +420,16 @@ public class RangePredicateEvaluatorFactory {
final boolean _includeUpperBoundary;
BytesRawValueBasedRangePredicateEvaluator(RangePredicate rangePredicate) {
- _lowerBoundary = BytesUtils.toBytes(rangePredicate.getLowerBoundary());
- _upperBoundary = BytesUtils.toBytes(rangePredicate.getUpperBoundary());
+ if (!"*".equals(rangePredicate.getLowerBoundary())) {
+ _lowerBoundary = BytesUtils.toBytes(rangePredicate.getLowerBoundary());
+ } else {
+ _lowerBoundary = null;
+ }
+ if (!"*".equals(rangePredicate.getUpperBoundary())) {
+ _upperBoundary = BytesUtils.toBytes(rangePredicate.getUpperBoundary());
+ } else {
+ _upperBoundary = null;
+ }
_includeLowerBoundary = rangePredicate.includeLowerBoundary();
_includeUpperBoundary = rangePredicate.includeUpperBoundary();
}
@@ -434,14 +442,14 @@ public class RangePredicateEvaluatorFactory {
@Override
public boolean applySV(byte[] value) {
boolean result = true;
- if (!_lowerBoundary.equals("*")) {
+ if (_lowerBoundary != null) {
if (_includeLowerBoundary) {
result = ByteArray.compare(_lowerBoundary, value) <= 0;
} else {
result = ByteArray.compare(_lowerBoundary, value) < 0;
}
}
- if (!_upperBoundary.equals("*")) {
+ if (_upperBoundary != null) {
if (_includeUpperBoundary) {
result &= ByteArray.compare(_upperBoundary, value) >= 0;
} else {
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryEqualsPredicateEvaluatorsTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryEqualsPredicateEvaluatorsTest.java
index 31b0220..823cdc3 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryEqualsPredicateEvaluatorsTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryEqualsPredicateEvaluatorsTest.java
@@ -24,6 +24,7 @@ import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.pinot.common.data.FieldSpec;
+import org.apache.pinot.common.utils.BytesUtils;
import org.apache.pinot.core.common.predicate.EqPredicate;
import org.apache.pinot.core.common.predicate.NEqPredicate;
import
org.apache.pinot.core.operator.filter.predicate.EqualsPredicateEvaluatorFactory;
@@ -219,4 +220,40 @@ public class NoDictionaryEqualsPredicateEvaluatorsTest {
!ArrayUtils.contains(randomStrings, stringValue));
}
}
+
+ @Test
+ public void testBytesPredicateEvaluators() {
+ byte[] bytesValue = RandomStringUtils.random(MAX_STRING_LENGTH).getBytes();
+ String hexStringValue = BytesUtils.toHexString(bytesValue);
+ EqPredicate eqPredicate = new EqPredicate(COLUMN_NAME,
Collections.singletonList(hexStringValue));
+ PredicateEvaluator eqPredicateEvaluator =
+ EqualsPredicateEvaluatorFactory.newRawValueBasedEvaluator(eqPredicate,
FieldSpec.DataType.BYTES);
+
+ NEqPredicate neqPredicate = new NEqPredicate(COLUMN_NAME,
Collections.singletonList(hexStringValue));
+ PredicateEvaluator neqPredicateEvaluator =
+
NotEqualsPredicateEvaluatorFactory.newRawValueBasedEvaluator(neqPredicate,
FieldSpec.DataType.BYTES);
+
+ Assert.assertTrue(eqPredicateEvaluator.applySV(bytesValue));
+ Assert.assertFalse(neqPredicateEvaluator.applySV(bytesValue));
+
+ byte[][] randomBytesArray = new byte[NUM_MULTI_VALUES][];
+ PredicateEvaluatorTestUtils.fillRandom(randomBytesArray,
MAX_STRING_LENGTH);
+ randomBytesArray[_random.nextInt(NUM_MULTI_VALUES)] = bytesValue;
+
+ Assert.assertTrue(eqPredicateEvaluator.applyMV(randomBytesArray,
NUM_MULTI_VALUES));
+ Assert.assertFalse(neqPredicateEvaluator.applyMV(randomBytesArray,
NUM_MULTI_VALUES));
+
+ for (int i = 0; i < 100; i++) {
+ byte[] randomBytes =
RandomStringUtils.random(MAX_STRING_LENGTH).getBytes();
+ String randomString = BytesUtils.toHexString(randomBytes);
+ Assert.assertEquals(eqPredicateEvaluator.applySV(randomBytes),
(randomString.equals(hexStringValue)));
+ Assert.assertEquals(neqPredicateEvaluator.applySV(randomBytes),
(!randomString.equals(hexStringValue)));
+
+ PredicateEvaluatorTestUtils.fillRandom(randomBytesArray,
MAX_STRING_LENGTH);
+ Assert.assertEquals(eqPredicateEvaluator.applyMV(randomBytesArray,
NUM_MULTI_VALUES),
+ ArrayUtils.contains(randomBytesArray, hexStringValue));
+ Assert.assertEquals(neqPredicateEvaluator.applyMV(randomBytesArray,
NUM_MULTI_VALUES),
+ !ArrayUtils.contains(randomBytesArray, hexStringValue));
+ }
+ }
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryInPredicateEvaluatorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryInPredicateEvaluatorTest.java
index 217f186..9435aaf 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryInPredicateEvaluatorTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryInPredicateEvaluatorTest.java
@@ -34,6 +34,7 @@ import java.util.Set;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.pinot.common.data.FieldSpec;
+import org.apache.pinot.common.utils.BytesUtils;
import org.apache.pinot.core.common.predicate.InPredicate;
import org.apache.pinot.core.common.predicate.NotInPredicate;
import
org.apache.pinot.core.operator.filter.predicate.InPredicateEvaluatorFactory;
@@ -252,4 +253,43 @@ public class NoDictionaryInPredicateEvaluatorTest {
Assert.assertTrue(inPredicateEvaluator.applyMV(multiValues,
NUM_MULTI_VALUES));
Assert.assertFalse(notInPredicateEvaluator.applyMV(multiValues,
NUM_MULTI_VALUES));
}
+
+ @Test
+ public void testBytesPredicateEvaluators() {
+ List<String> stringValues = new ArrayList<>(NUM_PREDICATE_VALUES);
+ Set<byte[]> valueSet = new HashSet<>();
+
+ for (int i = 0; i < 100; i++) {
+ byte[] value = RandomStringUtils.random(MAX_STRING_LENGTH).getBytes();
+ valueSet.add(value);
+ stringValues.add(BytesUtils.toHexString(value));
+ }
+
+ InPredicate inPredicate = new InPredicate(COLUMN_NAME, stringValues);
+ PredicateEvaluator inPredicateEvaluator =
+ InPredicateEvaluatorFactory.newRawValueBasedEvaluator(inPredicate,
FieldSpec.DataType.BYTES);
+
+ NotInPredicate notInPredicate = new NotInPredicate(COLUMN_NAME,
stringValues);
+ PredicateEvaluator notInPredicateEvaluator =
+
NotInPredicateEvaluatorFactory.newRawValueBasedEvaluator(notInPredicate,
FieldSpec.DataType.BYTES);
+
+ for (byte[] value : valueSet) {
+ Assert.assertTrue(inPredicateEvaluator.applySV(value));
+ Assert.assertFalse(notInPredicateEvaluator.applySV(value));
+ }
+
+ for (int i = 0; i < 100; i++) {
+ byte[] value = RandomStringUtils.random(MAX_STRING_LENGTH).getBytes();
+ Assert.assertEquals(inPredicateEvaluator.applySV(value),
valueSet.contains(value));
+ Assert.assertEquals(notInPredicateEvaluator.applySV(value),
!valueSet.contains(value));
+ }
+
+ byte[][] multiValues = new byte[NUM_MULTI_VALUES][];
+ PredicateEvaluatorTestUtils.fillRandom(multiValues, MAX_STRING_LENGTH);
+ multiValues[_random.nextInt(NUM_MULTI_VALUES)] =
+
BytesUtils.toBytes(stringValues.get(_random.nextInt(NUM_PREDICATE_VALUES)));
+
+ Assert.assertTrue(inPredicateEvaluator.applyMV(multiValues,
NUM_MULTI_VALUES));
+ Assert.assertFalse(notInPredicateEvaluator.applyMV(multiValues,
NUM_MULTI_VALUES));
+ }
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryRangePredicateEvaluatorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryRangePredicateEvaluatorTest.java
index 5f68646..94887a2 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryRangePredicateEvaluatorTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryRangePredicateEvaluatorTest.java
@@ -20,6 +20,8 @@ package org.apache.pinot.core.predicate;
import java.util.Collections;
import org.apache.pinot.common.data.FieldSpec;
+import org.apache.pinot.common.utils.BytesUtils;
+import org.apache.pinot.common.utils.primitive.ByteArray;
import org.apache.pinot.core.common.predicate.RangePredicate;
import org.apache.pinot.core.operator.filter.predicate.PredicateEvaluator;
import
org.apache.pinot.core.operator.filter.predicate.RangePredicateEvaluatorFactory;
@@ -261,6 +263,61 @@ public class NoDictionaryRangePredicateEvaluatorTest {
}
}
+ @Test
+ public void testBytesPredicateEvaluator() {
+
+ PredicateEvaluator predicateEvaluator = buildRangePredicate("[10\t\t20]",
FieldSpec.DataType.BYTES);
+ for (int i = 0x00; i < 0x30; i++) {
+ byte[] value = Integer.toString(i).getBytes();
+ Assert.assertEquals(predicateEvaluator.applySV(value),
+ (ByteArray.compare(value, new byte[]{0x10}) >= 0 &&
ByteArray.compare(value, new byte[]{0x20}) <= 0));
+ }
+
+ predicateEvaluator = buildRangePredicate("(10\t\t20]",
FieldSpec.DataType.BYTES);
+ for (int i = 0x00; i < 0x30; i++) {
+ byte[] value = Integer.toString(i).getBytes();
+ Assert.assertEquals(predicateEvaluator.applySV(value),
+ (ByteArray.compare(value, new byte[]{0x10}) > 0 &&
ByteArray.compare(value, new byte[]{0x20}) <= 0));
+ }
+
+ predicateEvaluator = buildRangePredicate("(10\t\t20)",
FieldSpec.DataType.BYTES);
+ for (int i = 0x00; i < 0x30; i++) {
+ byte[] value = Integer.toString(i).getBytes();
+ Assert.assertEquals(predicateEvaluator.applySV(value),
+ (ByteArray.compare(value, new byte[]{0x10}) > 0 &&
ByteArray.compare(value, new byte[]{0x20}) < 0));
+ }
+
+ predicateEvaluator = buildRangePredicate("(*\t\t10]",
FieldSpec.DataType.BYTES);
+ for (int i = 0x00; i < 0x30; i++) {
+ byte[] value = Integer.toString(i).getBytes();
+ Assert.assertEquals(predicateEvaluator.applySV(value),
ByteArray.compare(value, new byte[]{0x10}) <= 0);
+ }
+
+ predicateEvaluator = buildRangePredicate("(*\t\t10)",
FieldSpec.DataType.BYTES);
+ for (int i = 0x00; i < 0x30; i++) {
+ byte[] value = Integer.toString(i).getBytes();
+ Assert.assertEquals(predicateEvaluator.applySV(value),
ByteArray.compare(value, new byte[]{0x10}) < 0);
+ }
+
+ predicateEvaluator = buildRangePredicate("[10\t\t*)",
FieldSpec.DataType.BYTES);
+ for (int i = 0x00; i < 0x30; i++) {
+ byte[] value = Integer.toString(i).getBytes();
+ Assert.assertEquals(predicateEvaluator.applySV(value),
ByteArray.compare(value, new byte[]{0x10}) >= 0);
+ }
+
+ predicateEvaluator = buildRangePredicate("(10\t\t*)",
FieldSpec.DataType.BYTES);
+ for (int i = 0x00; i < 0x30; i++) {
+ byte[] value = Integer.toString(i).getBytes();
+ Assert.assertEquals(predicateEvaluator.applySV(value),
ByteArray.compare(value, new byte[]{0x10}) > 0);
+ }
+
+ predicateEvaluator = buildRangePredicate("(*\t\t*)",
FieldSpec.DataType.BYTES);
+ for (int i = 0x00; i < 0x30; i++) {
+ byte[] value = Integer.toString(i).getBytes();
+ Assert.assertTrue(predicateEvaluator.applySV(value));
+ }
+ }
+
private PredicateEvaluator buildRangePredicate(String rangeString,
FieldSpec.DataType dataType) {
RangePredicate predicate = new RangePredicate(COLUMN_NAME,
Collections.singletonList(rangeString));
return RangePredicateEvaluatorFactory.newRawValueBasedEvaluator(predicate,
dataType);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/predicate/PredicateEvaluatorTestUtils.java
b/pinot-core/src/test/java/org/apache/pinot/core/predicate/PredicateEvaluatorTestUtils.java
index 6af6e44..7f33f09 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/predicate/PredicateEvaluatorTestUtils.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/predicate/PredicateEvaluatorTestUtils.java
@@ -20,6 +20,7 @@ package org.apache.pinot.core.predicate;
import java.util.Random;
import org.apache.commons.lang.RandomStringUtils;
+import org.apache.pinot.common.utils.BytesUtils;
public class PredicateEvaluatorTestUtils {
@@ -54,4 +55,10 @@ public class PredicateEvaluatorTestUtils {
randomValues[i] = RandomStringUtils.random(maxStringLength);
}
}
+
+ public static void fillRandom(byte[][] randomValues, int maxStringLength) {
+ for (int i = 0; i < randomValues.length; i++) {
+ randomValues[i] = RandomStringUtils.random(maxStringLength).getBytes();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
