This is an automated email from the ASF dual-hosted git repository.
jihoonson pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 6674d72 Avoid sorting values in InDimFilter if possible (#9800)
6674d72 is described below
commit 6674d721bcc93ff959ae6d63c3eb71255179363d
Author: Jihoon Son <[email protected]>
AuthorDate: Wed May 6 15:26:36 2020 -0700
Avoid sorting values in InDimFilter if possible (#9800)
* Avoid sorting values in InDimFilter if possible
* tests
* more tests
* fix and and or filters
* fix build
* false and true vector matchers
* fix vector matchers
* checkstyle
* in filter null handling
* remove wrong test
* address comments
* remove unnecessary null check
* redundant separator
* address comments
* typo
* tests
---
.../apache/druid/common/config/NullHandling.java | 5 +
.../IngestSegmentFirehoseFactoryTimelineTest.java | 2 +-
.../main/java/org/apache/druid/query/Druids.java | 7 +-
.../apache/druid/query/filter/AndDimFilter.java | 17 ++-
.../org/apache/druid/query/filter/DimFilter.java | 4 +-
.../apache/druid/query/filter/DimFilterUtils.java | 1 +
.../{TrueDimFilter.java => FalseDimFilter.java} | 51 +++++--
.../org/apache/druid/query/filter/InDimFilter.java | 81 ++++++++----
.../apache/druid/query/filter/NotDimFilter.java | 9 +-
.../org/apache/druid/query/filter/OrDimFilter.java | 21 ++-
.../druid/query/filter/SelectorDimFilter.java | 2 +-
.../apache/druid/query/filter/TrueDimFilter.java | 36 ++++-
.../FalseVectorMatcher.java} | 36 ++---
.../TrueVectorMatcher.java} | 37 ++----
.../apache/druid/query/topn/TopNQueryBuilder.java | 7 +-
.../apache/druid/segment/filter/FalseFilter.java | 15 +++
.../org/apache/druid/segment/filter/InFilter.java | 5 +
.../druid/segment/filter/SelectorFilter.java | 5 +
.../apache/druid/segment/filter/TrueFilter.java | 15 +++
.../QueryableIndexVectorColumnSelectorFactory.java | 4 +-
.../vector/VectorColumnSelectorFactory.java | 10 +-
.../aggregation/FilteredAggregatorFactoryTest.java | 6 +-
.../druid/query/filter/AndDimFilterTest.java | 35 +++++
.../FalseDimFilterTest.java} | 38 +++---
.../druid/query/filter/InDimFilterSerDesrTest.java | 85 ------------
.../apache/druid/query/filter/InDimFilterTest.java | 147 +++++++++++++++++++++
.../apache/druid/query/filter/OrDimFilterTest.java | 35 +++++
.../TrueDimFilterTest.java} | 38 +++---
.../builtin/ArrayOverlapOperatorConversion.java | 3 +-
.../calcite/filtration/ConvertSelectorsToIns.java | 4 +-
.../druid/sql/calcite/filtration/Filtration.java | 12 +-
31 files changed, 532 insertions(+), 241 deletions(-)
diff --git
a/core/src/main/java/org/apache/druid/common/config/NullHandling.java
b/core/src/main/java/org/apache/druid/common/config/NullHandling.java
index bd0f0ee..51cb265 100644
--- a/core/src/main/java/org/apache/druid/common/config/NullHandling.java
+++ b/core/src/main/java/org/apache/druid/common/config/NullHandling.java
@@ -94,6 +94,11 @@ public class NullHandling
//CHECKSTYLE.ON: Regexp
}
+ public static boolean needsEmptyToNull(@Nullable String value)
+ {
+ return replaceWithDefault() && Strings.isNullOrEmpty(value);
+ }
+
@Nullable
public static String defaultStringValue()
{
diff --git
a/indexing-service/src/test/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactoryTimelineTest.java
b/indexing-service/src/test/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactoryTimelineTest.java
index 7c2760c..e8dff5d 100644
---
a/indexing-service/src/test/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactoryTimelineTest.java
+++
b/indexing-service/src/test/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactoryTimelineTest.java
@@ -348,7 +348,7 @@ public class IngestSegmentFirehoseFactoryTimelineTest
DATA_SOURCE,
testCase.interval,
null,
- new TrueDimFilter(),
+ TrueDimFilter.instance(),
Arrays.asList(DIMENSIONS),
Arrays.asList(METRICS),
// Split as much as possible
diff --git a/processing/src/main/java/org/apache/druid/query/Druids.java
b/processing/src/main/java/org/apache/druid/query/Druids.java
index 1dcd076..522f26b 100644
--- a/processing/src/main/java/org/apache/druid/query/Druids.java
+++ b/processing/src/main/java/org/apache/druid/query/Druids.java
@@ -23,7 +23,7 @@ import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.granularity.Granularity;
import org.apache.druid.query.aggregation.AggregatorFactory;
@@ -58,6 +58,7 @@ import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
*/
@@ -202,7 +203,9 @@ public class Druids
public TimeseriesQueryBuilder filters(String dimensionName, String value,
String... values)
{
- dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values),
null, null);
+ final Set<String> filterValues = Sets.newHashSet(values);
+ filterValues.add(value);
+ dimFilter = new InDimFilter(dimensionName, filterValues, null, null);
return this;
}
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/AndDimFilter.java
b/processing/src/main/java/org/apache/druid/query/filter/AndDimFilter.java
index 682c247..18df9d6 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/AndDimFilter.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/AndDimFilter.java
@@ -33,6 +33,7 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
/**
*/
@@ -72,8 +73,20 @@ public class AndDimFilter implements DimFilter
@Override
public DimFilter optimize()
{
- List<DimFilter> elements = DimFilters.optimize(fields);
- return elements.size() == 1 ? elements.get(0) : new AndDimFilter(elements);
+ List<DimFilter> elements = DimFilters.optimize(fields)
+ .stream()
+ .filter(filter -> !(filter instanceof
TrueDimFilter))
+ .collect(Collectors.toList());
+ if (elements.isEmpty()) {
+ // All elements were TrueDimFilter after optimization
+ return TrueDimFilter.instance();
+ } else if (elements.size() == 1) {
+ return elements.get(0);
+ } else if (elements.stream().anyMatch(filter -> filter instanceof
FalseDimFilter)) {
+ return FalseDimFilter.instance();
+ } else {
+ return new AndDimFilter(elements);
+ }
}
@Override
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java
b/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java
index 7688d36..6a38457 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java
@@ -47,7 +47,8 @@ import java.util.Set;
@JsonSubTypes.Type(name = "interval", value = IntervalDimFilter.class),
@JsonSubTypes.Type(name = "like", value = LikeDimFilter.class),
@JsonSubTypes.Type(name = "expression", value = ExpressionDimFilter.class),
- @JsonSubTypes.Type(name = "true", value = TrueDimFilter.class)
+ @JsonSubTypes.Type(name = "true", value = TrueDimFilter.class),
+ @JsonSubTypes.Type(name = "false", value = FalseDimFilter.class)
})
public interface DimFilter extends Cacheable
{
@@ -78,6 +79,7 @@ public interface DimFilter extends Cacheable
* @return a RangeSet that represent the possible range of the input
dimension, or null if it is not possible to
* determine for this DimFilter.
*/
+ @Nullable
RangeSet<String> getDimensionRangeSet(String dimension);
/**
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/DimFilterUtils.java
b/processing/src/main/java/org/apache/druid/query/filter/DimFilterUtils.java
index c980adb..3fcd719 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/DimFilterUtils.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/DimFilterUtils.java
@@ -52,6 +52,7 @@ public class DimFilterUtils
static final byte COLUMN_COMPARISON_CACHE_ID = 0xD;
static final byte EXPRESSION_CACHE_ID = 0xE;
static final byte TRUE_CACHE_ID = 0xF;
+ static final byte FALSE_CACHE_ID = 0x11;
public static final byte BLOOM_DIM_FILTER_CACHE_ID = 0x10;
public static final byte STRING_SEPARATOR = (byte) 0xFF;
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java
b/processing/src/main/java/org/apache/druid/query/filter/FalseDimFilter.java
similarity index 59%
copy from
processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java
copy to
processing/src/main/java/org/apache/druid/query/filter/FalseDimFilter.java
index 2254358..4b21e5e 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/FalseDimFilter.java
@@ -19,21 +19,29 @@
package org.apache.druid.query.filter;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.google.common.collect.ImmutableRangeSet;
import com.google.common.collect.RangeSet;
import org.apache.druid.query.cache.CacheKeyBuilder;
-import org.apache.druid.segment.filter.TrueFilter;
+import org.apache.druid.segment.filter.FalseFilter;
+import javax.annotation.Nullable;
import java.util.Collections;
import java.util.Set;
-/**
- */
-public class TrueDimFilter implements DimFilter
+public class FalseDimFilter implements DimFilter
{
- @Override
- public byte[] getCacheKey()
+ private static final FalseDimFilter INSTANCE = new FalseDimFilter();
+ private static final byte[] CACHE_KEY = new
CacheKeyBuilder(DimFilterUtils.FALSE_CACHE_ID).build();
+
+ @JsonCreator
+ public static FalseDimFilter instance()
+ {
+ return INSTANCE;
+ }
+
+ private FalseDimFilter()
{
- return new CacheKeyBuilder(DimFilterUtils.TRUE_CACHE_ID).build();
}
@Override
@@ -45,13 +53,14 @@ public class TrueDimFilter implements DimFilter
@Override
public Filter toFilter()
{
- return TrueFilter.instance();
+ return FalseFilter.instance();
}
+ @Nullable
@Override
public RangeSet<String> getDimensionRangeSet(String dimension)
{
- return null;
+ return ImmutableRangeSet.of();
}
@Override
@@ -59,4 +68,28 @@ public class TrueDimFilter implements DimFilter
{
return Collections.emptySet();
}
+
+ @Override
+ public byte[] getCacheKey()
+ {
+ return CACHE_KEY;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return DimFilterUtils.FALSE_CACHE_ID;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ return o != null && o.getClass() == this.getClass();
+ }
+
+ @Override
+ public String toString()
+ {
+ return "FALSE";
+ }
}
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java
b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java
index d97d086..d5ec558 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java
@@ -20,6 +20,7 @@
package org.apache.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
@@ -29,16 +30,18 @@ import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Ordering;
import com.google.common.collect.Range;
import com.google.common.collect.RangeSet;
import com.google.common.collect.TreeRangeSet;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.guava.Comparators;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.query.lookup.LookupExtractionFn;
@@ -47,14 +50,16 @@ import org.apache.druid.segment.DimensionHandlerUtils;
import org.apache.druid.segment.filter.InFilter;
import javax.annotation.Nullable;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
+import java.util.stream.Collectors;
public class InDimFilter implements DimFilter
{
@@ -63,7 +68,7 @@ public class InDimFilter implements DimFilter
public static final int NUMERIC_HASHING_THRESHOLD = 16;
// Values can contain `null` object
- private final SortedSet<String> values;
+ private final Set<String> values;
private final String dimension;
@Nullable
private final ExtractionFn extractionFn;
@@ -73,10 +78,15 @@ public class InDimFilter implements DimFilter
private final Supplier<DruidFloatPredicate> floatPredicateSupplier;
private final Supplier<DruidDoublePredicate> doublePredicateSupplier;
+ @JsonIgnore
+ private byte[] cacheKey;
+
@JsonCreator
public InDimFilter(
@JsonProperty("dimension") String dimension,
- @JsonProperty("values") Collection<String> values,
+ // This 'values' collection instance can be reused if possible to avoid
copying a big collection.
+ // Callers should _not_ modify the collection after it is passed to this
constructor.
+ @JsonProperty("values") Set<String> values,
@JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn,
@JsonProperty("filterTuning") @Nullable FilterTuning filterTuning
)
@@ -84,9 +94,12 @@ public class InDimFilter implements DimFilter
Preconditions.checkNotNull(dimension, "dimension can not be null");
Preconditions.checkArgument(values != null, "values can not be null");
- this.values = new TreeSet<>(Comparators.naturalNullsFirst());
- for (String value : values) {
- this.values.add(NullHandling.emptyToNullIfNeeded(value));
+ // The values set can be huge. Try to avoid copying the set if possible.
+ // Note that we may still need to copy values to a list for caching. See
getCacheKey().
+ if ((NullHandling.sqlCompatible() ||
values.stream().noneMatch(NullHandling::needsEmptyToNull))) {
+ this.values = values;
+ } else {
+ this.values =
values.stream().map(NullHandling::emptyToNullIfNeeded).collect(Collectors.toSet());
}
this.dimension = dimension;
this.extractionFn = extractionFn;
@@ -96,10 +109,13 @@ public class InDimFilter implements DimFilter
this.doublePredicateSupplier = getDoublePredicateSupplier();
}
+ /**
+ * This constructor should be called only in unit tests since it creates a
new hash set wrapping the given values.
+ */
@VisibleForTesting
public InDimFilter(String dimension, Collection<String> values, @Nullable
ExtractionFn extractionFn)
{
- this(dimension, values, extractionFn, null);
+ this(dimension, new HashSet<>(values), extractionFn, null);
}
@JsonProperty
@@ -115,6 +131,7 @@ public class InDimFilter implements DimFilter
}
@Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
@JsonProperty
public ExtractionFn getExtractionFn()
{
@@ -132,31 +149,40 @@ public class InDimFilter implements DimFilter
@Override
public byte[] getCacheKey()
{
- boolean hasNull = false;
- for (String value : values) {
- if (value == null) {
- hasNull = true;
- break;
+ if (cacheKey == null) {
+ final List<String> sortedValues = new ArrayList<>(values);
+ sortedValues.sort(Comparator.nullsFirst(Ordering.natural()));
+ final Hasher hasher = Hashing.sha256().newHasher();
+ for (String v : sortedValues) {
+ if (v == null) {
+ hasher.putInt(0);
+ } else {
+ hasher.putString(v, StandardCharsets.UTF_8);
+ }
}
+ cacheKey = new CacheKeyBuilder(DimFilterUtils.IN_CACHE_ID)
+ .appendString(dimension)
+ .appendByte(DimFilterUtils.STRING_SEPARATOR)
+ .appendByteArray(extractionFn == null ? new byte[0] :
extractionFn.getCacheKey())
+ .appendByte(DimFilterUtils.STRING_SEPARATOR)
+ .appendByteArray(hasher.hash().asBytes())
+ .build();
}
- return new CacheKeyBuilder(DimFilterUtils.IN_CACHE_ID)
- .appendString(dimension)
- .appendByte(DimFilterUtils.STRING_SEPARATOR)
- .appendByteArray(extractionFn == null ? new byte[0] :
extractionFn.getCacheKey())
- .appendByte(DimFilterUtils.STRING_SEPARATOR)
- .appendByte(hasNull ? NullHandling.IS_NULL_BYTE :
NullHandling.IS_NOT_NULL_BYTE)
- .appendByte(DimFilterUtils.STRING_SEPARATOR)
- .appendStrings(values).build();
+ return cacheKey;
}
@Override
public DimFilter optimize()
{
InDimFilter inFilter = optimizeLookup();
+
+ if (inFilter.values.isEmpty()) {
+ return FalseDimFilter.instance();
+ }
if (inFilter.values.size() == 1) {
return new SelectorDimFilter(
inFilter.dimension,
- inFilter.values.first(),
+ inFilter.values.iterator().next(),
inFilter.getExtractionFn(),
filterTuning
);
@@ -171,7 +197,7 @@ public class InDimFilter implements DimFilter
LookupExtractionFn exFn = (LookupExtractionFn) extractionFn;
LookupExtractor lookup = exFn.getLookup();
- final List<String> keys = new ArrayList<>();
+ final Set<String> keys = new HashSet<>();
for (String value : values) {
// We cannot do an unapply()-based optimization if the selector value
@@ -215,6 +241,7 @@ public class InDimFilter implements DimFilter
);
}
+ @Nullable
@Override
public RangeSet<String> getDimensionRangeSet(String dimension)
{
@@ -302,7 +329,7 @@ public class InDimFilter implements DimFilter
// This supplier must be thread-safe, since this DimFilter will be accessed
in the query runners.
private Supplier<DruidLongPredicate> getLongPredicateSupplier()
{
- Supplier<DruidLongPredicate> longPredicate = () -> createLongPredicate();
+ Supplier<DruidLongPredicate> longPredicate = this::createLongPredicate;
return Suppliers.memoize(longPredicate);
}
@@ -330,7 +357,7 @@ public class InDimFilter implements DimFilter
private Supplier<DruidFloatPredicate> getFloatPredicateSupplier()
{
- Supplier<DruidFloatPredicate> floatPredicate = () ->
createFloatPredicate();
+ Supplier<DruidFloatPredicate> floatPredicate = this::createFloatPredicate;
return Suppliers.memoize(floatPredicate);
}
@@ -358,7 +385,7 @@ public class InDimFilter implements DimFilter
private Supplier<DruidDoublePredicate> getDoublePredicateSupplier()
{
- Supplier<DruidDoublePredicate> doublePredicate = () ->
createDoublePredicate();
+ Supplier<DruidDoublePredicate> doublePredicate =
this::createDoublePredicate;
return Suppliers.memoize(doublePredicate);
}
}
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/NotDimFilter.java
b/processing/src/main/java/org/apache/druid/query/filter/NotDimFilter.java
index 039667e..0cd2969 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/NotDimFilter.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/NotDimFilter.java
@@ -60,10 +60,17 @@ public class NotDimFilter implements DimFilter
return ByteBuffer.allocate(1 +
subKey.length).put(DimFilterUtils.NOT_CACHE_ID).put(subKey).array();
}
+ @SuppressWarnings("ObjectEquality")
@Override
public DimFilter optimize()
{
- return new NotDimFilter(this.getField().optimize());
+ final DimFilter optimized = this.getField().optimize();
+ if (optimized == FalseDimFilter.instance()) {
+ return TrueDimFilter.instance();
+ } else if (optimized == TrueDimFilter.instance()) {
+ return FalseDimFilter.instance();
+ }
+ return new NotDimFilter(optimized);
}
@Override
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/OrDimFilter.java
b/processing/src/main/java/org/apache/druid/query/filter/OrDimFilter.java
index 762ad72..ca8c105 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/OrDimFilter.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/OrDimFilter.java
@@ -34,6 +34,7 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
/**
*/
@@ -44,9 +45,7 @@ public class OrDimFilter implements DimFilter
private final List<DimFilter> fields;
@JsonCreator
- public OrDimFilter(
- @JsonProperty("fields") List<DimFilter> fields
- )
+ public OrDimFilter(@JsonProperty("fields") List<DimFilter> fields)
{
fields = DimFilters.filterNulls(fields);
Preconditions.checkArgument(fields.size() > 0, "OR operator requires at
least one field");
@@ -82,8 +81,20 @@ public class OrDimFilter implements DimFilter
@Override
public DimFilter optimize()
{
- List<DimFilter> elements = DimFilters.optimize(fields);
- return elements.size() == 1 ? elements.get(0) : new OrDimFilter(elements);
+ List<DimFilter> elements = DimFilters.optimize(fields)
+ .stream()
+ .filter(filter -> !(filter instanceof
FalseDimFilter))
+ .collect(Collectors.toList());
+ if (elements.isEmpty()) {
+ // All elements were FalseDimFilter after optimization
+ return FalseDimFilter.instance();
+ } else if (elements.size() == 1) {
+ return elements.get(0);
+ } else if (elements.stream().anyMatch(filter -> filter instanceof
TrueDimFilter)) {
+ return TrueDimFilter.instance();
+ } else {
+ return new OrDimFilter(elements);
+ }
}
@Override
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java
b/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java
index 91ec334..d27f413 100644
---
a/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java
+++
b/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java
@@ -96,7 +96,7 @@ public class SelectorDimFilter implements DimFilter
@Override
public DimFilter optimize()
{
- return new InDimFilter(dimension, Collections.singletonList(value),
extractionFn, filterTuning).optimize();
+ return new InDimFilter(dimension, Collections.singleton(value),
extractionFn, filterTuning).optimize();
}
@Override
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java
b/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java
index 2254358..af29710 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java
@@ -19,6 +19,7 @@
package org.apache.druid.query.filter;
+import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.common.collect.RangeSet;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.segment.filter.TrueFilter;
@@ -26,14 +27,25 @@ import org.apache.druid.segment.filter.TrueFilter;
import java.util.Collections;
import java.util.Set;
-/**
- */
public class TrueDimFilter implements DimFilter
{
+ private static final TrueDimFilter INSTANCE = new TrueDimFilter();
+ private static final byte[] CACHE_KEY = new
CacheKeyBuilder(DimFilterUtils.TRUE_CACHE_ID).build();
+
+ @JsonCreator
+ public static TrueDimFilter instance()
+ {
+ return INSTANCE;
+ }
+
+ private TrueDimFilter()
+ {
+ }
+
@Override
public byte[] getCacheKey()
{
- return new CacheKeyBuilder(DimFilterUtils.TRUE_CACHE_ID).build();
+ return CACHE_KEY;
}
@Override
@@ -59,4 +71,22 @@ public class TrueDimFilter implements DimFilter
{
return Collections.emptySet();
}
+
+ @Override
+ public int hashCode()
+ {
+ return DimFilterUtils.TRUE_CACHE_ID;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ return o != null && o.getClass() == this.getClass();
+ }
+
+ @Override
+ public String toString()
+ {
+ return "TRUE";
+ }
}
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java
b/processing/src/main/java/org/apache/druid/query/filter/vector/FalseVectorMatcher.java
similarity index 56%
copy from
processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java
copy to
processing/src/main/java/org/apache/druid/query/filter/vector/FalseVectorMatcher.java
index 2254358..9d20c1d 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java
+++
b/processing/src/main/java/org/apache/druid/query/filter/vector/FalseVectorMatcher.java
@@ -17,46 +17,34 @@
* under the License.
*/
-package org.apache.druid.query.filter;
+package org.apache.druid.query.filter.vector;
-import com.google.common.collect.RangeSet;
-import org.apache.druid.query.cache.CacheKeyBuilder;
-import org.apache.druid.segment.filter.TrueFilter;
+import org.apache.druid.segment.vector.VectorSizeInspector;
-import java.util.Collections;
-import java.util.Set;
-
-/**
- */
-public class TrueDimFilter implements DimFilter
+public class FalseVectorMatcher implements VectorValueMatcher
{
- @Override
- public byte[] getCacheKey()
- {
- return new CacheKeyBuilder(DimFilterUtils.TRUE_CACHE_ID).build();
- }
+ private final VectorSizeInspector vectorSizeInspector;
- @Override
- public DimFilter optimize()
+ public FalseVectorMatcher(VectorSizeInspector vectorSizeInspector)
{
- return this;
+ this.vectorSizeInspector = vectorSizeInspector;
}
@Override
- public Filter toFilter()
+ public ReadableVectorMatch match(ReadableVectorMatch mask)
{
- return TrueFilter.instance();
+ return VectorMatch.allFalse();
}
@Override
- public RangeSet<String> getDimensionRangeSet(String dimension)
+ public int getMaxVectorSize()
{
- return null;
+ return vectorSizeInspector.getMaxVectorSize();
}
@Override
- public Set<String> getRequiredColumns()
+ public int getCurrentVectorSize()
{
- return Collections.emptySet();
+ return vectorSizeInspector.getCurrentVectorSize();
}
}
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java
b/processing/src/main/java/org/apache/druid/query/filter/vector/TrueVectorMatcher.java
similarity index 56%
copy from
processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java
copy to
processing/src/main/java/org/apache/druid/query/filter/vector/TrueVectorMatcher.java
index 2254358..24b2b27 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java
+++
b/processing/src/main/java/org/apache/druid/query/filter/vector/TrueVectorMatcher.java
@@ -17,46 +17,35 @@
* under the License.
*/
-package org.apache.druid.query.filter;
+package org.apache.druid.query.filter.vector;
-import com.google.common.collect.RangeSet;
-import org.apache.druid.query.cache.CacheKeyBuilder;
-import org.apache.druid.segment.filter.TrueFilter;
+import org.apache.druid.segment.vector.VectorSizeInspector;
-import java.util.Collections;
-import java.util.Set;
-
-/**
- */
-public class TrueDimFilter implements DimFilter
+public class TrueVectorMatcher implements VectorValueMatcher
{
- @Override
- public byte[] getCacheKey()
- {
- return new CacheKeyBuilder(DimFilterUtils.TRUE_CACHE_ID).build();
- }
+ private final VectorSizeInspector vectorSizeInspector;
- @Override
- public DimFilter optimize()
+ public TrueVectorMatcher(VectorSizeInspector vectorSizeInspector)
{
- return this;
+ this.vectorSizeInspector = vectorSizeInspector;
}
@Override
- public Filter toFilter()
+ public ReadableVectorMatch match(ReadableVectorMatch mask)
{
- return TrueFilter.instance();
+ // The given mask is all true for its valid selections.
+ return mask;
}
@Override
- public RangeSet<String> getDimensionRangeSet(String dimension)
+ public int getMaxVectorSize()
{
- return null;
+ return vectorSizeInspector.getMaxVectorSize();
}
@Override
- public Set<String> getRequiredColumns()
+ public int getCurrentVectorSize()
{
- return Collections.emptySet();
+ return vectorSizeInspector.getCurrentVectorSize();
}
}
diff --git
a/processing/src/main/java/org/apache/druid/query/topn/TopNQueryBuilder.java
b/processing/src/main/java/org/apache/druid/query/topn/TopNQueryBuilder.java
index d50c565..c6715df 100644
--- a/processing/src/main/java/org/apache/druid/query/topn/TopNQueryBuilder.java
+++ b/processing/src/main/java/org/apache/druid/query/topn/TopNQueryBuilder.java
@@ -19,7 +19,7 @@
package org.apache.druid.query.topn;
-import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.granularity.Granularity;
import org.apache.druid.query.DataSource;
@@ -42,6 +42,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* A Builder for TopNQuery.
@@ -230,7 +231,9 @@ public class TopNQueryBuilder
public TopNQueryBuilder filters(String dimensionName, String value,
String... values)
{
- dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values),
null, null);
+ final Set<String> filterValues = Sets.newHashSet(values);
+ filterValues.add(value);
+ dimFilter = new InDimFilter(dimensionName, filterValues, null, null);
return this;
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/filter/FalseFilter.java
b/processing/src/main/java/org/apache/druid/segment/filter/FalseFilter.java
index e143b70..902dae4 100644
--- a/processing/src/main/java/org/apache/druid/segment/filter/FalseFilter.java
+++ b/processing/src/main/java/org/apache/druid/segment/filter/FalseFilter.java
@@ -23,8 +23,11 @@ import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.ValueMatcher;
+import org.apache.druid.query.filter.vector.FalseVectorMatcher;
+import org.apache.druid.query.filter.vector.VectorValueMatcher;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
+import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import java.util.Collections;
import java.util.Set;
@@ -61,6 +64,12 @@ public class FalseFilter implements Filter
}
@Override
+ public VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory
factory)
+ {
+ return new FalseVectorMatcher(factory.getVectorSizeInspector());
+ }
+
+ @Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return true;
@@ -79,6 +88,12 @@ public class FalseFilter implements Filter
}
@Override
+ public boolean canVectorizeMatcher()
+ {
+ return true;
+ }
+
+ @Override
public Set<String> getRequiredColumns()
{
return Collections.emptySet();
diff --git
a/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java
b/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java
index 6e1da91..d609a54 100644
--- a/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java
+++ b/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java
@@ -54,6 +54,11 @@ import java.util.Set;
* given {@link #values}.
* For multi-valued dimension, this filter returns true if one of the
dimension values matches to one of the
* given {@link #values}.
+ *
+ * In SQL-compatible null handling mode, this filter is equivalent to {@code
(dimension IN [values])} or
+ * {@code (dimension IN [non-null values] OR dimension IS NULL)} when {@link
#values} contains nulls.
+ * In default null handling mode, this filter is equivalent to {@code
(dimension IN [values])} or
+ * {@code (dimension IN [non-null values, ''])} when {@link #values} contains
nulls.
*/
public class InFilter implements Filter
{
diff --git
a/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java
b/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java
index 7587427..f56bfac 100644
---
a/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java
+++
b/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java
@@ -38,6 +38,11 @@ import java.util.Objects;
import java.util.Set;
/**
+ * This filter is to select the rows where the {@link #dimension} has the
{@link #value}. The value can be null.
+ * In SQL-compatible null handling mode, this filter is equivalent to {@code
dimension = value}
+ * or {@code dimension IS NULL} when the value is null.
+ * In default null handling mode, this filter is equivalent to {@code
dimension = value} or
+ * {@code dimension = ''} when the value is null.
*/
public class SelectorFilter implements Filter
{
diff --git
a/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java
b/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java
index 58cda8c..bb35b4f 100644
--- a/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java
+++ b/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java
@@ -23,8 +23,11 @@ import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.BitmapIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.ValueMatcher;
+import org.apache.druid.query.filter.vector.TrueVectorMatcher;
+import org.apache.druid.query.filter.vector.VectorValueMatcher;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
+import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import java.util.Collections;
import java.util.Set;
@@ -57,6 +60,12 @@ public class TrueFilter implements Filter
}
@Override
+ public VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory
factory)
+ {
+ return new TrueVectorMatcher(factory.getVectorSizeInspector());
+ }
+
+ @Override
public boolean supportsBitmapIndex(BitmapIndexSelector selector)
{
return true;
@@ -75,6 +84,12 @@ public class TrueFilter implements Filter
}
@Override
+ public boolean canVectorizeMatcher()
+ {
+ return true;
+ }
+
+ @Override
public Set<String> getRequiredColumns()
{
return Collections.emptySet();
diff --git
a/processing/src/main/java/org/apache/druid/segment/vector/QueryableIndexVectorColumnSelectorFactory.java
b/processing/src/main/java/org/apache/druid/segment/vector/QueryableIndexVectorColumnSelectorFactory.java
index d816589..b9cfe4f 100644
---
a/processing/src/main/java/org/apache/druid/segment/vector/QueryableIndexVectorColumnSelectorFactory.java
+++
b/processing/src/main/java/org/apache/druid/segment/vector/QueryableIndexVectorColumnSelectorFactory.java
@@ -66,9 +66,9 @@ public class QueryableIndexVectorColumnSelectorFactory
implements VectorColumnSe
}
@Override
- public int getMaxVectorSize()
+ public VectorSizeInspector getVectorSizeInspector()
{
- return offset.getMaxVectorSize();
+ return offset;
}
@Override
diff --git
a/processing/src/main/java/org/apache/druid/segment/vector/VectorColumnSelectorFactory.java
b/processing/src/main/java/org/apache/druid/segment/vector/VectorColumnSelectorFactory.java
index 1634cc6..bea845f 100644
---
a/processing/src/main/java/org/apache/druid/segment/vector/VectorColumnSelectorFactory.java
+++
b/processing/src/main/java/org/apache/druid/segment/vector/VectorColumnSelectorFactory.java
@@ -32,11 +32,19 @@ import javax.annotation.Nullable;
public interface VectorColumnSelectorFactory
{
/**
+ * Returns a {@link VectorSizeInspector} for the {@link VectorCursor} that
generated this object.
+ */
+ VectorSizeInspector getVectorSizeInspector();
+
+ /**
* Returns the maximum vector size for the {@link VectorCursor} that
generated this object.
*
* @see VectorCursor#getMaxVectorSize()
*/
- int getMaxVectorSize();
+ default int getMaxVectorSize()
+ {
+ return getVectorSizeInspector().getMaxVectorSize();
+ }
/**
* Returns a string-typed, single-value-per-row column selector.
diff --git
a/processing/src/test/java/org/apache/druid/query/aggregation/FilteredAggregatorFactoryTest.java
b/processing/src/test/java/org/apache/druid/query/aggregation/FilteredAggregatorFactoryTest.java
index 4db7b0e..879a252 100644
---
a/processing/src/test/java/org/apache/druid/query/aggregation/FilteredAggregatorFactoryTest.java
+++
b/processing/src/test/java/org/apache/druid/query/aggregation/FilteredAggregatorFactoryTest.java
@@ -30,17 +30,17 @@ public class FilteredAggregatorFactoryTest
{
Assert.assertEquals("overrideName", new FilteredAggregatorFactory(
new CountAggregatorFactory("foo"),
- new TrueDimFilter(),
+ TrueDimFilter.instance(),
"overrideName"
).getName());
Assert.assertEquals("delegateName", new FilteredAggregatorFactory(
new CountAggregatorFactory("delegateName"),
- new TrueDimFilter(),
+ TrueDimFilter.instance(),
""
).getName());
Assert.assertEquals("delegateName", new FilteredAggregatorFactory(
new CountAggregatorFactory("delegateName"),
- new TrueDimFilter(),
+ TrueDimFilter.instance(),
null
).getName());
}
diff --git
a/processing/src/test/java/org/apache/druid/query/filter/AndDimFilterTest.java
b/processing/src/test/java/org/apache/druid/query/filter/AndDimFilterTest.java
index 7aeae01..e45ed29 100644
---
a/processing/src/test/java/org/apache/druid/query/filter/AndDimFilterTest.java
+++
b/processing/src/test/java/org/apache/druid/query/filter/AndDimFilterTest.java
@@ -65,4 +65,39 @@ public class AndDimFilterTest extends
InitializedNullHandlingTest
);
Assert.assertEquals(expected, dimFilter.toFilter());
}
+
+ @Test
+ public void testOptimieShortCircuitWithFalseFilter()
+ {
+ DimFilter filter = DimFilterTestUtils.and(
+ DimFilterTestUtils.selector("col1", "1"),
+ FalseDimFilter.instance()
+ );
+ Assert.assertTrue(filter.optimize() instanceof FalseDimFilter);
+ }
+
+ @Test
+ public void testOptimizeOringTrueFilters()
+ {
+ DimFilter filter = DimFilterTestUtils.and(TrueDimFilter.instance(),
TrueDimFilter.instance());
+ Assert.assertSame(TrueDimFilter.instance(), filter.optimize());
+ }
+
+ @Test
+ public void testOptimizeAndOfSingleFilterUnwrapAnd()
+ {
+ DimFilter expected = DimFilterTestUtils.selector("col1", "1");
+ DimFilter filter = DimFilterTestUtils.and(expected);
+ Assert.assertEquals(expected, filter.optimize());
+ }
+
+ @Test
+ public void testOptimizeAndOfMultipleFiltersReturningAsItIs()
+ {
+ DimFilter filter = DimFilterTestUtils.and(
+ DimFilterTestUtils.selector("col1", "1"),
+ DimFilterTestUtils.selector("col1", "2")
+ );
+ Assert.assertEquals(filter, filter.optimize());
+ }
}
diff --git
a/processing/src/test/java/org/apache/druid/query/aggregation/FilteredAggregatorFactoryTest.java
b/processing/src/test/java/org/apache/druid/query/filter/FalseDimFilterTest.java
similarity index 54%
copy from
processing/src/test/java/org/apache/druid/query/aggregation/FilteredAggregatorFactoryTest.java
copy to
processing/src/test/java/org/apache/druid/query/filter/FalseDimFilterTest.java
index 4db7b0e..a512bca 100644
---
a/processing/src/test/java/org/apache/druid/query/aggregation/FilteredAggregatorFactoryTest.java
+++
b/processing/src/test/java/org/apache/druid/query/filter/FalseDimFilterTest.java
@@ -17,31 +17,31 @@
* under the License.
*/
-package org.apache.druid.query.aggregation;
+package org.apache.druid.query.filter;
-import org.apache.druid.query.filter.TrueDimFilter;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.apache.druid.jackson.DefaultObjectMapper;
import org.junit.Assert;
import org.junit.Test;
-public class FilteredAggregatorFactoryTest
+import java.io.IOException;
+
+public class FalseDimFilterTest
{
@Test
- public void testSimpleNaming()
+ public void testSerde() throws IOException
+ {
+ final ObjectMapper mapper = new DefaultObjectMapper();
+ final FalseDimFilter original = FalseDimFilter.instance();
+ final byte[] bytes = mapper.writeValueAsBytes(original);
+ final FalseDimFilter fromBytes = (FalseDimFilter) mapper.readValue(bytes,
DimFilter.class);
+ Assert.assertSame(original, fromBytes);
+ }
+
+ @Test
+ public void testEquals()
{
- Assert.assertEquals("overrideName", new FilteredAggregatorFactory(
- new CountAggregatorFactory("foo"),
- new TrueDimFilter(),
- "overrideName"
- ).getName());
- Assert.assertEquals("delegateName", new FilteredAggregatorFactory(
- new CountAggregatorFactory("delegateName"),
- new TrueDimFilter(),
- ""
- ).getName());
- Assert.assertEquals("delegateName", new FilteredAggregatorFactory(
- new CountAggregatorFactory("delegateName"),
- new TrueDimFilter(),
- null
- ).getName());
+ EqualsVerifier.forClass(FalseDimFilter.class).usingGetClass().verify();
}
}
diff --git
a/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java
b/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java
deleted file mode 100644
index 691769a..0000000
---
a/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.filter;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.inject.Injector;
-import com.google.inject.Key;
-import org.apache.druid.guice.GuiceInjectors;
-import org.apache.druid.guice.annotations.Json;
-import org.apache.druid.query.extraction.RegexDimExtractionFn;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-
-public class InDimFilterSerDesrTest
-{
- private static ObjectMapper mapper;
-
- private final String serializedFilter =
-
"{\"type\":\"in\",\"dimension\":\"dimTest\",\"values\":[\"bad\",\"good\"],\"extractionFn\":null}";
-
- @Before
- public void setUp()
- {
- Injector defaultInjector = GuiceInjectors.makeStartupInjector();
- mapper = defaultInjector.getInstance(Key.get(ObjectMapper.class,
Json.class));
- }
-
- @Test
- public void testDeserialization() throws IOException
- {
- final InDimFilter actualInDimFilter =
mapper.readerFor(DimFilter.class).readValue(serializedFilter);
- final InDimFilter expectedInDimFilter = new InDimFilter("dimTest",
Arrays.asList("good", "bad"), null);
- Assert.assertEquals(expectedInDimFilter, actualInDimFilter);
- }
-
- @Test
- public void testSerialization() throws IOException
- {
- final InDimFilter dimInFilter = new InDimFilter("dimTest",
Arrays.asList("good", "bad"), null);
- final String actualSerializedFilter =
mapper.writeValueAsString(dimInFilter);
- Assert.assertEquals(serializedFilter, actualSerializedFilter);
- }
-
- @Test
- public void testGetCacheKey()
- {
- final InDimFilter inDimFilter_1 = new InDimFilter("dimTest",
Arrays.asList("good", "bad"), null);
- final InDimFilter inDimFilter_2 = new InDimFilter("dimTest",
Collections.singletonList("good,bad"), null);
- Assert.assertNotEquals(inDimFilter_1.getCacheKey(),
inDimFilter_2.getCacheKey());
-
- RegexDimExtractionFn regexFn = new RegexDimExtractionFn(".*", false, null);
- final InDimFilter inDimFilter_3 = new InDimFilter("dimTest",
Arrays.asList("good", "bad"), regexFn);
- final InDimFilter inDimFilter_4 = new InDimFilter("dimTest",
Collections.singletonList("good,bad"), regexFn);
- Assert.assertNotEquals(inDimFilter_3.getCacheKey(),
inDimFilter_4.getCacheKey());
- }
-
- @Test
- public void testGetCacheKeyNullValue() throws IOException
- {
- InDimFilter inDimFilter =
mapper.readValue("{\"type\":\"in\",\"dimension\":\"dimTest\",\"values\":[null]}",
InDimFilter.class);
- Assert.assertNotNull(inDimFilter.getCacheKey());
- }
-}
diff --git
a/processing/src/test/java/org/apache/druid/query/filter/InDimFilterTest.java
b/processing/src/test/java/org/apache/druid/query/filter/InDimFilterTest.java
new file mode 100644
index 0000000..9cf5b8f
--- /dev/null
+++
b/processing/src/test/java/org/apache/druid/query/filter/InDimFilterTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query.filter;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.query.extraction.RegexDimExtractionFn;
+import org.apache.druid.testing.InitializedNullHandlingTest;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Set;
+
+public class InDimFilterTest extends InitializedNullHandlingTest
+{
+ private ObjectMapper mapper = new DefaultObjectMapper();
+
+ private final String serializedFilter =
+
"{\"type\":\"in\",\"dimension\":\"dimTest\",\"values\":[\"bad\",\"good\"]}";
+
+ @Test
+ public void testDeserialization() throws IOException
+ {
+ final InDimFilter actualInDimFilter =
mapper.readerFor(DimFilter.class).readValue(serializedFilter);
+ final InDimFilter expectedInDimFilter = new InDimFilter("dimTest",
Arrays.asList("good", "bad"), null);
+ Assert.assertEquals(expectedInDimFilter, actualInDimFilter);
+ }
+
+ @Test
+ public void testSerialization() throws IOException
+ {
+ final InDimFilter dimInFilter = new InDimFilter("dimTest",
Arrays.asList("good", "bad"), null);
+ final String actualSerializedFilter =
mapper.writeValueAsString(dimInFilter);
+ Assert.assertEquals(serializedFilter, actualSerializedFilter);
+ }
+
+ @Test
+ public void testGetValuesWithValuesSetOfNonEmptyStringsUseTheGivenSet()
+ {
+ final Set<String> values = ImmutableSet.of("v1", "v2", "v3");
+ final InDimFilter filter = new InDimFilter("dim", values, null, null);
+ Assert.assertSame(values, filter.getValues());
+ }
+
+ @Test
+ public void testGetValuesWithValuesSetIncludingEmptyString()
+ {
+ final Set<String> values = Sets.newHashSet("v1", "", "v3");
+ final InDimFilter filter = new InDimFilter("dim", values, null, null);
+ if (NullHandling.replaceWithDefault()) {
+ Assert.assertNotSame(values, filter.getValues());
+ Assert.assertEquals(Sets.newHashSet("v1", null, "v3"),
filter.getValues());
+ } else {
+ Assert.assertSame(values, filter.getValues());
+ }
+ }
+
+ @Test
+ public void testGetCacheKeyReturningSameKeyForValuesOfDifferentOrders()
+ {
+ final InDimFilter dimFilter1 = new InDimFilter("dim",
ImmutableList.of("v1", "v2"), null);
+ final InDimFilter dimFilter2 = new InDimFilter("dim",
ImmutableList.of("v2", "v1"), null);
+ Assert.assertArrayEquals(dimFilter1.getCacheKey(),
dimFilter2.getCacheKey());
+ }
+
+ @Test
+ public void
testGetCacheKeyDifferentKeysForListOfStringsAndSingleStringOfLists()
+ {
+ final InDimFilter inDimFilter1 = new InDimFilter("dimTest",
Arrays.asList("good", "bad"), null);
+ final InDimFilter inDimFilter2 = new InDimFilter("dimTest",
Collections.singletonList("good,bad"), null);
+ Assert.assertFalse(Arrays.equals(inDimFilter1.getCacheKey(),
inDimFilter2.getCacheKey()));
+ }
+
+ @Test
+ public void
testGetCacheKeyDifferentKeysForListOfStringsAndSingleStringOfListsWithExtractFn()
+ {
+ RegexDimExtractionFn regexFn = new RegexDimExtractionFn(".*", false, null);
+ final InDimFilter inDimFilter1 = new InDimFilter("dimTest",
Arrays.asList("good", "bad"), regexFn);
+ final InDimFilter inDimFilter2 = new InDimFilter("dimTest",
Collections.singletonList("good,bad"), regexFn);
+ Assert.assertFalse(Arrays.equals(inDimFilter1.getCacheKey(),
inDimFilter2.getCacheKey()));
+ }
+
+ @Test
+ public void testGetCacheKeyNullValue() throws IOException
+ {
+ InDimFilter inDimFilter = mapper.readValue(
+ "{\"type\":\"in\",\"dimension\":\"dimTest\",\"values\":[null]}",
+ InDimFilter.class
+ );
+ Assert.assertNotNull(inDimFilter.getCacheKey());
+ }
+
+ @Test
+ public void testGetCacheKeyReturningDifferentKeysWithAndWithoutNull()
+ {
+ InDimFilter filter1 = new InDimFilter("dim", Arrays.asList("val", null),
null);
+ InDimFilter filter2 = new InDimFilter("dim",
Collections.singletonList("val"), null);
+ Assert.assertFalse(Arrays.equals(filter1.getCacheKey(),
filter2.getCacheKey()));
+ }
+
+ @Test
+ public void testGetCacheKeyReturningCachedCacheKey()
+ {
+ final InDimFilter filter = new InDimFilter("dim", ImmutableList.of("v1",
"v2"), null);
+ // Compares the array object, not the elements of the array
+ Assert.assertSame(filter.getCacheKey(), filter.getCacheKey());
+ }
+
+ @Test
+ public void
testGetDimensionRangeSetValuesOfDifferentOrdersReturningSameResult()
+ {
+ final InDimFilter dimFilter1 = new InDimFilter("dim",
ImmutableList.of("v1", "v2", "v3"), null);
+ final InDimFilter dimFilter2 = new InDimFilter("dim",
ImmutableList.of("v3", "v2", "v1"), null);
+ Assert.assertEquals(dimFilter1.getDimensionRangeSet("dim"),
dimFilter2.getDimensionRangeSet("dim"));
+ }
+
+ @Test
+ public void testOptimizeSingleValueInToSelector()
+ {
+ final InDimFilter filter = new InDimFilter("dim",
Collections.singleton("v1"), null);
+ Assert.assertEquals(new SelectorDimFilter("dim", "v1", null),
filter.optimize());
+ }
+}
diff --git
a/processing/src/test/java/org/apache/druid/query/filter/OrDimFilterTest.java
b/processing/src/test/java/org/apache/druid/query/filter/OrDimFilterTest.java
index 9f02194..fe080db 100644
---
a/processing/src/test/java/org/apache/druid/query/filter/OrDimFilterTest.java
+++
b/processing/src/test/java/org/apache/druid/query/filter/OrDimFilterTest.java
@@ -50,4 +50,39 @@ public class OrDimFilterTest extends
InitializedNullHandlingTest
);
Assert.assertEquals(expected, dimFilter.toFilter());
}
+
+ @Test
+ public void testOptimieShortCircuitWithTrueFilter()
+ {
+ DimFilter filter = DimFilterTestUtils.or(
+ DimFilterTestUtils.selector("col1", "1"),
+ TrueDimFilter.instance()
+ );
+ Assert.assertTrue(filter.optimize() instanceof TrueDimFilter);
+ }
+
+ @Test
+ public void testOptimizeOringFalseFilters()
+ {
+ DimFilter filter = DimFilterTestUtils.or(FalseDimFilter.instance(),
FalseDimFilter.instance());
+ Assert.assertSame(FalseDimFilter.instance(), filter.optimize());
+ }
+
+ @Test
+ public void testOptimizeOrOfSingleFilterUnwrapOr()
+ {
+ DimFilter expected = DimFilterTestUtils.selector("col1", "1");
+ DimFilter filter = DimFilterTestUtils.or(expected);
+ Assert.assertEquals(expected, filter.optimize());
+ }
+
+ @Test
+ public void testOptimizeOrOfMultipleFiltersReturningAsItIs()
+ {
+ DimFilter filter = DimFilterTestUtils.or(
+ DimFilterTestUtils.selector("col1", "1"),
+ DimFilterTestUtils.selector("col1", "2")
+ );
+ Assert.assertEquals(filter, filter.optimize());
+ }
}
diff --git
a/processing/src/test/java/org/apache/druid/query/aggregation/FilteredAggregatorFactoryTest.java
b/processing/src/test/java/org/apache/druid/query/filter/TrueDimFilterTest.java
similarity index 54%
copy from
processing/src/test/java/org/apache/druid/query/aggregation/FilteredAggregatorFactoryTest.java
copy to
processing/src/test/java/org/apache/druid/query/filter/TrueDimFilterTest.java
index 4db7b0e..3c88382 100644
---
a/processing/src/test/java/org/apache/druid/query/aggregation/FilteredAggregatorFactoryTest.java
+++
b/processing/src/test/java/org/apache/druid/query/filter/TrueDimFilterTest.java
@@ -17,31 +17,31 @@
* under the License.
*/
-package org.apache.druid.query.aggregation;
+package org.apache.druid.query.filter;
-import org.apache.druid.query.filter.TrueDimFilter;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.apache.druid.jackson.DefaultObjectMapper;
import org.junit.Assert;
import org.junit.Test;
-public class FilteredAggregatorFactoryTest
+import java.io.IOException;
+
+public class TrueDimFilterTest
{
@Test
- public void testSimpleNaming()
+ public void testSerde() throws IOException
+ {
+ final ObjectMapper mapper = new DefaultObjectMapper();
+ final TrueDimFilter original = TrueDimFilter.instance();
+ final byte[] bytes = mapper.writeValueAsBytes(original);
+ final TrueDimFilter fromBytes = (TrueDimFilter) mapper.readValue(bytes,
DimFilter.class);
+ Assert.assertSame(original, fromBytes);
+ }
+
+ @Test
+ public void testEquals()
{
- Assert.assertEquals("overrideName", new FilteredAggregatorFactory(
- new CountAggregatorFactory("foo"),
- new TrueDimFilter(),
- "overrideName"
- ).getName());
- Assert.assertEquals("delegateName", new FilteredAggregatorFactory(
- new CountAggregatorFactory("delegateName"),
- new TrueDimFilter(),
- ""
- ).getName());
- Assert.assertEquals("delegateName", new FilteredAggregatorFactory(
- new CountAggregatorFactory("delegateName"),
- new TrueDimFilter(),
- null
- ).getName());
+ EqualsVerifier.forClass(TrueDimFilter.class).usingGetClass().verify();
}
}
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOverlapOperatorConversion.java
b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOverlapOperatorConversion.java
index 175670e..cb1ddf5 100644
---
a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOverlapOperatorConversion.java
+++
b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOverlapOperatorConversion.java
@@ -123,7 +123,8 @@ public class ArrayOverlapOperatorConversion extends
BaseExpressionDimFilterOpera
return new InDimFilter(
simpleExtractionExpr.getSimpleExtraction().getColumn(),
Sets.newHashSet(arrayElements),
- simpleExtractionExpr.getSimpleExtraction().getExtractionFn()
+ simpleExtractionExpr.getSimpleExtraction().getExtractionFn(),
+ null
);
}
}
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java
b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java
index 83da057..54b2625 100644
---
a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java
+++
b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java
@@ -20,6 +20,7 @@
package org.apache.druid.sql.calcite.filtration;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.query.filter.DimFilter;
import org.apache.druid.query.filter.InDimFilter;
@@ -33,6 +34,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
public class ConvertSelectorsToIns extends BottomUpTransform
{
@@ -78,7 +80,7 @@ public class ConvertSelectorsToIns extends BottomUpTransform
final List<SelectorDimFilter> filterList = entry.getValue();
if (filterList.size() > 1) {
// We found a simplification. Remove the old filters and add new
ones.
- final List<String> values = new ArrayList<>();
+ final Set<String> values =
Sets.newHashSetWithExpectedSize(filterList.size());
for (final SelectorDimFilter selector : filterList) {
values.add(selector.getValue());
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Filtration.java
b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Filtration.java
index 677961e..8f2f760 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Filtration.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Filtration.java
@@ -23,9 +23,9 @@ import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.query.filter.DimFilter;
-import org.apache.druid.query.filter.ExpressionDimFilter;
+import org.apache.druid.query.filter.FalseDimFilter;
+import org.apache.druid.query.filter.TrueDimFilter;
import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
import org.apache.druid.query.spec.QuerySegmentSpec;
import org.apache.druid.segment.column.RowSignature;
@@ -36,12 +36,8 @@ import java.util.Objects;
public class Filtration
{
- private static final DimFilter MATCH_NOTHING = new ExpressionDimFilter(
- "1 == 2", ExprMacroTable.nil()
- );
- private static final DimFilter MATCH_EVERYTHING = new ExpressionDimFilter(
- "1 == 1", ExprMacroTable.nil()
- );
+ private static final DimFilter MATCH_NOTHING = FalseDimFilter.instance();
+ private static final DimFilter MATCH_EVERYTHING = TrueDimFilter.instance();
// 1) If "dimFilter" is null, it should be ignored and not affect filtration.
// 2) There is an implicit AND between "intervals" and "dimFilter" (if
dimFilter is non-null).
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
