This is an automated email from the ASF dual-hosted git repository.
jonwei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new b06ac54 add PrefixFilteredDimensionSpec for multi-value dimensions
(#6307)
b06ac54 is described below
commit b06ac54a5ee9176a97baad9d69ce96aa27265eae
Author: dongyifeng <[email protected]>
AuthorDate: Sat Oct 13 08:51:09 2018 +0800
add PrefixFilteredDimensionSpec for multi-value dimensions (#6307)
* add PrefixFilteredDimensionSpec for multi-value dimensions
* add docs for PrefixFilteredDimensionSpec
* remove unnecessary null handling
* add null check to the result of NullHandling
---
docs/content/querying/dimensionspecs.md | 6 ++
.../druid/query/dimension/DimensionSpec.java | 3 +-
...nSpec.java => PrefixFilteredDimensionSpec.java} | 49 +++++------
.../dimension/RegexFilteredDimensionSpec.java | 3 +-
.../dimension/PrefixFilteredDimensionSpecTest.java | 98 ++++++++++++++++++++++
5 files changed, 130 insertions(+), 29 deletions(-)
diff --git a/docs/content/querying/dimensionspecs.md
b/docs/content/querying/dimensionspecs.md
index b70fa68..e9a41e0 100644
--- a/docs/content/querying/dimensionspecs.md
+++ b/docs/content/querying/dimensionspecs.md
@@ -66,6 +66,12 @@ Following filtered dimension spec retains only the values
matching regex. Note t
{ "type" : "regexFiltered", "delegate" : <dimensionSpec>, "pattern": <java
regex pattern> }
```
+Following filtered dimension spec retains only the values starting with the
same prefix.
+
+```json
+{ "type" : "prefixFiltered", "delegate" : <dimensionSpec>, "prefix": <prefix
string> }
+```
+
For more details and examples, see [multi-value
dimensions](multi-value-dimensions.html).
### Lookup DimensionSpecs
diff --git
a/processing/src/main/java/org/apache/druid/query/dimension/DimensionSpec.java
b/processing/src/main/java/org/apache/druid/query/dimension/DimensionSpec.java
index 3adf5d5..bbf970e 100644
---
a/processing/src/main/java/org/apache/druid/query/dimension/DimensionSpec.java
+++
b/processing/src/main/java/org/apache/druid/query/dimension/DimensionSpec.java
@@ -37,7 +37,8 @@ import javax.annotation.Nullable;
@JsonSubTypes.Type(name = "default", value = DefaultDimensionSpec.class),
@JsonSubTypes.Type(name = "extraction", value =
ExtractionDimensionSpec.class),
@JsonSubTypes.Type(name = "regexFiltered", value =
RegexFilteredDimensionSpec.class),
- @JsonSubTypes.Type(name = "listFiltered", value =
ListFilteredDimensionSpec.class)
+ @JsonSubTypes.Type(name = "listFiltered", value =
ListFilteredDimensionSpec.class),
+ @JsonSubTypes.Type(name = "prefixFiltered", value =
PrefixFilteredDimensionSpec.class)
})
public interface DimensionSpec extends Cacheable
{
diff --git
a/processing/src/main/java/org/apache/druid/query/dimension/RegexFilteredDimensionSpec.java
b/processing/src/main/java/org/apache/druid/query/dimension/PrefixFilteredDimensionSpec.java
similarity index 74%
copy from
processing/src/main/java/org/apache/druid/query/dimension/RegexFilteredDimensionSpec.java
copy to
processing/src/main/java/org/apache/druid/query/dimension/PrefixFilteredDimensionSpec.java
index 0ad457d..d4904b5 100644
---
a/processing/src/main/java/org/apache/druid/query/dimension/RegexFilteredDimensionSpec.java
+++
b/processing/src/main/java/org/apache/druid/query/dimension/PrefixFilteredDimensionSpec.java
@@ -22,42 +22,37 @@ package org.apache.druid.query.dimension;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
-import it.unimi.dsi.fastutil.ints.Int2IntMap;
-import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.filter.DimFilterUtils;
import org.apache.druid.segment.DimensionSelector;
-
-import javax.annotation.Nullable;
+import it.unimi.dsi.fastutil.ints.Int2IntMap;
+import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
import java.nio.ByteBuffer;
-import java.util.regex.Pattern;
+import javax.annotation.Nullable;
/**
*/
-public class RegexFilteredDimensionSpec extends BaseFilteredDimensionSpec
+public class PrefixFilteredDimensionSpec extends BaseFilteredDimensionSpec
{
- private static final byte CACHE_TYPE_ID = 0x2;
+ private static final byte CACHE_TYPE_ID = 0x4;
- private final String pattern;
+ private final String prefix;
- private final Pattern compiledRegex;
-
- public RegexFilteredDimensionSpec(
+ public PrefixFilteredDimensionSpec(
@JsonProperty("delegate") DimensionSpec delegate,
- @JsonProperty("pattern") String pattern //rows not matching the pattern
will be discarded
+ @JsonProperty("prefix") String prefix //rows not starting with the
prefix will be discarded
)
{
super(delegate);
- this.pattern = Preconditions.checkNotNull(pattern, "pattern must not be
null");
- this.compiledRegex = Pattern.compile(pattern);
+ this.prefix = Preconditions.checkNotNull(prefix, "prefix must not be
null");
}
@JsonProperty
- public String getPattern()
+ public String getPrefix()
{
- return pattern;
+ return prefix;
}
@Override
@@ -76,7 +71,8 @@ public class RegexFilteredDimensionSpec extends
BaseFilteredDimensionSpec
@Override
public boolean apply(@Nullable String input)
{
- return
compiledRegex.matcher(NullHandling.nullToEmptyIfNeeded(input)).matches();
+ String val = NullHandling.nullToEmptyIfNeeded(input);
+ return val == null ? false : val.startsWith(prefix);
}
}
);
@@ -87,7 +83,7 @@ public class RegexFilteredDimensionSpec extends
BaseFilteredDimensionSpec
forwardMapping.defaultReturnValue(-1);
for (int i = 0; i < selectorCardinality; i++) {
String val = NullHandling.nullToEmptyIfNeeded(selector.lookupName(i));
- if (val != null && compiledRegex.matcher(val).matches()) {
+ if (val != null && val.startsWith(prefix)) {
forwardMapping.put(i, count++);
}
}
@@ -103,12 +99,12 @@ public class RegexFilteredDimensionSpec extends
BaseFilteredDimensionSpec
public byte[] getCacheKey()
{
byte[] delegateCacheKey = delegate.getCacheKey();
- byte[] regexBytes = StringUtils.toUtf8(pattern);
- return ByteBuffer.allocate(2 + delegateCacheKey.length + regexBytes.length)
+ byte[] prefixBytes = StringUtils.toUtf8(prefix);
+ return ByteBuffer.allocate(2 + delegateCacheKey.length +
prefixBytes.length)
.put(CACHE_TYPE_ID)
.put(delegateCacheKey)
.put(DimFilterUtils.STRING_SEPARATOR)
- .put(regexBytes)
+ .put(prefixBytes)
.array();
}
@@ -122,28 +118,27 @@ public class RegexFilteredDimensionSpec extends
BaseFilteredDimensionSpec
return false;
}
- RegexFilteredDimensionSpec that = (RegexFilteredDimensionSpec) o;
+ PrefixFilteredDimensionSpec that = (PrefixFilteredDimensionSpec) o;
if (!delegate.equals(that.delegate)) {
return false;
}
- return pattern.equals(that.pattern);
-
+ return prefix.equals(that.prefix);
}
@Override
public int hashCode()
{
int result = delegate.hashCode();
- result = 31 * result + pattern.hashCode();
+ result = 31 * result + prefix.hashCode();
return result;
}
@Override
public String toString()
{
- return "RegexFilteredDimensionSpec{" +
- "pattern='" + pattern + '\'' +
+ return "PrefixFilteredDimensionSpec{" +
+ "Prefix='" + prefix + '\'' +
'}';
}
}
diff --git
a/processing/src/main/java/org/apache/druid/query/dimension/RegexFilteredDimensionSpec.java
b/processing/src/main/java/org/apache/druid/query/dimension/RegexFilteredDimensionSpec.java
index 0ad457d..84883c6 100644
---
a/processing/src/main/java/org/apache/druid/query/dimension/RegexFilteredDimensionSpec.java
+++
b/processing/src/main/java/org/apache/druid/query/dimension/RegexFilteredDimensionSpec.java
@@ -76,7 +76,8 @@ public class RegexFilteredDimensionSpec extends
BaseFilteredDimensionSpec
@Override
public boolean apply(@Nullable String input)
{
- return
compiledRegex.matcher(NullHandling.nullToEmptyIfNeeded(input)).matches();
+ String val = NullHandling.nullToEmptyIfNeeded(input);
+ return val == null ? false :
compiledRegex.matcher(val).matches();
}
}
);
diff --git
a/processing/src/test/java/org/apache/druid/query/dimension/PrefixFilteredDimensionSpecTest.java
b/processing/src/test/java/org/apache/druid/query/dimension/PrefixFilteredDimensionSpecTest.java
new file mode 100644
index 0000000..2dc8fec
--- /dev/null
+++
b/processing/src/test/java/org/apache/druid/query/dimension/PrefixFilteredDimensionSpecTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query.dimension;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.druid.segment.DimensionSelector;
+import org.apache.druid.segment.TestHelper;
+import org.apache.druid.segment.data.IndexedInts;
+import java.util.Arrays;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ */
+public class PrefixFilteredDimensionSpecTest
+{
+
+ @Test
+ public void testSerde() throws Exception
+ {
+ ObjectMapper mapper = TestHelper.makeJsonMapper();
+
+ String jsonStr = "{\n"
+ + " \"type\": \"prefixFiltered\",\n"
+ + " \"delegate\": {\n"
+ + " \"type\": \"default\",\n"
+ + " \"dimension\": \"foo\",\n"
+ + " \"outputName\": \"bar\"\n"
+ + " },\n"
+ + " \"prefix\": \"xxx\"\n"
+ + "}";
+
+ PrefixFilteredDimensionSpec actual = (PrefixFilteredDimensionSpec)
mapper.readValue(
+ mapper.writeValueAsString(mapper.readValue(jsonStr,
DimensionSpec.class)),
+ DimensionSpec.class);
+
+ PrefixFilteredDimensionSpec expected = new PrefixFilteredDimensionSpec(
+ new DefaultDimensionSpec("foo", "bar"),
+ "xxx"
+ );
+
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testGetCacheKey()
+ {
+ PrefixFilteredDimensionSpec spec1 = new PrefixFilteredDimensionSpec(
+ new DefaultDimensionSpec("foo", "bar"),
+ "xxx"
+ );
+
+ PrefixFilteredDimensionSpec spec2 = new PrefixFilteredDimensionSpec(
+ new DefaultDimensionSpec("foo", "bar"),
+ "xyz"
+ );
+
+ Assert.assertFalse(Arrays.equals(spec1.getCacheKey(),
spec2.getCacheKey()));
+ }
+
+ @Test
+ public void testDecorator()
+ {
+ PrefixFilteredDimensionSpec spec = new PrefixFilteredDimensionSpec(
+ new DefaultDimensionSpec("foo", "far"),
+ "c"
+ );
+
+ DimensionSelector selector = spec.decorate(TestDimensionSelector.instance);
+
+ Assert.assertEquals(1, selector.getValueCardinality());
+
+ IndexedInts row = selector.getRow();
+ Assert.assertEquals(1, row.size());
+ Assert.assertEquals(0, row.get(0));
+
+ Assert.assertEquals("c", selector.lookupName(0));
+
+ Assert.assertEquals(0, selector.idLookup().lookupId("c"));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
