gianm commented on code in PR #16039:
URL: https://github.com/apache/druid/pull/16039#discussion_r1535024793
##########
processing/src/main/java/org/apache/druid/query/filter/TypedInFilter.java:
##########
@@ -222,12 +254,9 @@ public RangeSet<String> getDimensionRangeSet(String
dimension)
return null;
}
RangeSet<String> retSet = TreeRangeSet.create();
- for (Object value : lazyMatchValues.get()) {
- String valueEquivalent =
NullHandling.nullToEmptyIfNeeded(Evals.asString(value));
+ for (Object value : sortedMatchValues.get()) {
+ String valueEquivalent = Evals.asString(value);
if (valueEquivalent == null) {
- // Case when SQL compatible null handling is enabled
- // Range.singleton(null) is invalid, so use the fact that
Review Comment:
Most of this comment is still useful in explaining why we're doing
`lessThan("")`.
##########
processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnAndIndexSupplier.java:
##########
@@ -258,6 +265,70 @@ public <T> T computeBitmapResult(BitmapResultFactory<T>
bitmapResultFactory, boo
}
}
+ private final class DoubleValueSetIndexes implements ValueSetIndexes
+ {
+ final FixedIndexed<Double> dictionary = doubleDictionarySupplier.get();
+
+ @Nullable
+ @Override
+ public BitmapColumnIndex forSortedValues(@Nonnull List<?> sortedValues,
TypeSignature<ValueType> matchValueType)
+ {
+ final boolean matchNull = sortedValues.get(0) == null;
+ final Supplier<ImmutableBitmap> unknownsIndex = () -> {
+ if (!matchNull && dictionary.get(0) == null) {
+ return valueIndexes.get(0);
+ }
+ return null;
+ };
+
+ // values are doubles and ordered in double order
+ if (matchValueType.is(ValueType.DOUBLE)) {
Review Comment:
I am okay with what you decide here. I can see both ways making sense.
##########
processing/src/main/java/org/apache/druid/query/filter/TypedInFilter.java:
##########
@@ -0,0 +1,742 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query.filter;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
+import com.google.common.collect.RangeSet;
+import com.google.common.collect.Sets;
+import com.google.common.collect.TreeRangeSet;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+import it.unimi.dsi.fastutil.doubles.DoubleOpenHashSet;
+import it.unimi.dsi.fastutil.floats.FloatOpenHashSet;
+import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
+import it.unimi.dsi.fastutil.objects.ObjectRBTreeSet;
+import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.error.InvalidInput;
+import org.apache.druid.java.util.common.IAE;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.math.expr.Evals;
+import org.apache.druid.query.cache.CacheKeyBuilder;
+import org.apache.druid.query.filter.vector.VectorValueMatcher;
+import
org.apache.druid.query.filter.vector.VectorValueMatcherColumnProcessorFactory;
+import org.apache.druid.segment.ColumnInspector;
+import org.apache.druid.segment.ColumnProcessors;
+import org.apache.druid.segment.ColumnSelectorFactory;
+import org.apache.druid.segment.DimensionHandlerUtils;
+import org.apache.druid.segment.column.ColumnIndexSupplier;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.NullableTypeStrategy;
+import org.apache.druid.segment.column.ValueType;
+import org.apache.druid.segment.filter.Filters;
+import org.apache.druid.segment.index.BitmapColumnIndex;
+import org.apache.druid.segment.index.semantic.Utf8ValueSetIndexes;
+import org.apache.druid.segment.index.semantic.ValueSetIndexes;
+import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
+
+import javax.annotation.Nullable;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Approximately like the SQL 'IN' filter, with the main difference being that
this will match NULL values if contained
+ * in the values list instead of ignoring them.
+ * <p>
+ * This is a typed version of {@link InDimFilter}, which allows the match
values to exist in their native type and
+ * sorted in their type native order for better performance matching against
all column types.
+ */
+public class TypedInFilter extends AbstractOptimizableDimFilter implements
Filter
+{
+ /**
+ * Column to match {@link #sortedMatchValues} or {@link
#sortedUtf8MatchValueBytes} against.
+ */
+ private final String column;
+
+ /**
+ * Type of values contained in {@link #sortedMatchValues}. This might be the
same or different than the
+ * {@link ColumnType} of {@link #column}, but is encouraged to be the same
there are several optimizations available
+ * if they match.
+ */
+ private final ColumnType matchValueType;
+
+ /**
+ * Unsorted values. This will be null if the values are found to be sorted,
or have been already sorted "upstream".
+ * Otherwise, this set of values will be lazily computed into {@link
#sortedMatchValues} as needed, e.g. for
+ * JSON serialization, cache key building, building a hashcode, or checking
equality.
+ */
+ @Nullable
+ private final List<?> unsortedValues;
+
+ /**
+ * Supplier for list of values sorted by {@link #matchValueType}. This is
lazily computed if
+ * {@link #unsortedValues} is not null and previously sorted.
Review Comment:
would be useful to include a comment about whether this can contain
duplicates or not, and if duplicates are not meant to be here, then also
include a test that validates that deduplication happens on `values` (even if
they are provided in sorted order).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
