[CALCITE-1655] Druid adapter: add IN filter (Slim Bouguerra) We cannot do end-to-end tests of IN and BETWEEN because Calcite expands these before creating RexNode, but this change contains unit tests of converting RexCall with IN and BETWEEN into Druid JSON.
Close apache/calcite#381 Project: http://git-wip-us.apache.org/repos/asf/calcite/repo Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/f8546915 Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/f8546915 Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/f8546915 Branch: refs/heads/master Commit: f8546915671cc4d96fa3b5561e289c3bfd80209f Parents: 4e10382 Author: Slim Bouguerra <[email protected]> Authored: Thu Feb 23 16:49:47 2017 -0800 Committer: Julian Hyde <[email protected]> Committed: Sat Mar 4 13:02:45 2017 -0800 ---------------------------------------------------------------------- druid/pom.xml | 5 + .../calcite/adapter/druid/DruidQuery.java | 77 ++++++++--- .../adapter/druid/DruidQueryFilterTest.java | 127 +++++++++++++++++++ pom.xml | 6 +- 4 files changed, 196 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite/blob/f8546915/druid/pom.xml ---------------------------------------------------------------------- diff --git a/druid/pom.xml b/druid/pom.xml index 4191d3d..68e4d92 100644 --- a/druid/pom.xml +++ b/druid/pom.xml @@ -95,6 +95,11 @@ limitations under the License. <artifactId>slf4j-log4j12</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/calcite/blob/f8546915/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java ---------------------------------------------------------------------- diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java index 694f9c7..74c0cc0 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java @@ -66,6 +66,7 @@ import org.apache.calcite.util.Util; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; @@ -865,7 +866,8 @@ public class DruidQuery extends AbstractRelNode implements BindableRel { } /** Translates scalar expressions to Druid field references. */ - private static class Translator { + @VisibleForTesting + protected static class Translator { final List<String> dimensions = new ArrayList<>(); final List<String> metrics = new ArrayList<>(); final DruidTable druidTable; @@ -917,7 +919,7 @@ public class DruidQuery extends AbstractRelNode implements BindableRel { } } - @SuppressWarnings("incomplete-switch") private JsonFilter translateFilter(RexNode e) { + private JsonFilter translateFilter(RexNode e) { final RexCall call; switch (e.getKind()) { case EQUALS: @@ -926,6 +928,8 @@ public class DruidQuery extends AbstractRelNode implements BindableRel { case GREATER_THAN_OR_EQUAL: case LESS_THAN: case LESS_THAN_OR_EQUAL: + case IN: + case BETWEEN: call = (RexCall) e; int posRef; int posConstant; @@ -938,34 +942,50 @@ public class DruidQuery extends AbstractRelNode implements BindableRel { } else { throw new AssertionError("it is not a valid comparison: " + e); } + final boolean numeric = + call.getOperands().get(posRef).getType().getFamily() + == SqlTypeFamily.NUMERIC; switch (e.getKind()) { case EQUALS: return new JsonSelector("selector", tr(e, posRef), tr(e, posConstant)); case NOT_EQUALS: return new JsonCompositeFilter("not", - ImmutableList.of(new JsonSelector("selector", tr(e, posRef), tr(e, posConstant)))); + new JsonSelector("selector", tr(e, posRef), tr(e, posConstant))); case GREATER_THAN: - return new JsonBound("bound", tr(e, posRef), tr(e, posConstant), true, null, false, - call.getOperands().get(posRef).getType().getFamily() == SqlTypeFamily.NUMERIC); + return new JsonBound("bound", tr(e, posRef), tr(e, posConstant), + true, null, false, numeric); case GREATER_THAN_OR_EQUAL: - return new JsonBound("bound", tr(e, posRef), tr(e, posConstant), false, null, false, - call.getOperands().get(posRef).getType().getFamily() == SqlTypeFamily.NUMERIC); + return new JsonBound("bound", tr(e, posRef), tr(e, posConstant), + false, null, false, numeric); case LESS_THAN: - return new JsonBound("bound", tr(e, posRef), null, false, tr(e, posConstant), true, - call.getOperands().get(posRef).getType().getFamily() == SqlTypeFamily.NUMERIC); + return new JsonBound("bound", tr(e, posRef), null, false, + tr(e, posConstant), true, numeric); case LESS_THAN_OR_EQUAL: - return new JsonBound("bound", tr(e, posRef), null, false, tr(e, posConstant), false, - call.getOperands().get(posRef).getType().getFamily() == SqlTypeFamily.NUMERIC); + return new JsonBound("bound", tr(e, posRef), null, false, + tr(e, posConstant), false, numeric); + case IN: + ImmutableList.Builder<String> listBuilder = ImmutableList.builder(); + for (RexNode rexNode: call.getOperands()) { + if (rexNode.getKind() == SqlKind.LITERAL) { + listBuilder.add(((RexLiteral) rexNode).getValue2().toString()); + } + } + return new JsonInFilter("in", tr(e, posRef), listBuilder.build()); + case BETWEEN: + return new JsonBound("bound", tr(e, posRef), tr(e, 2), false, + tr(e, 3), false, numeric); + default: + throw new AssertionError(); } - break; case AND: case OR: case NOT: call = (RexCall) e; return new JsonCompositeFilter(e.getKind().lowerName, translateFilters(call.getOperands())); + default: + throw new AssertionError("cannot translate filter: " + e); } - throw new AssertionError("cannot translate filter: " + e); } private String tr(RexNode call, int index) { @@ -1164,7 +1184,8 @@ public class DruidQuery extends AbstractRelNode implements BindableRel { } /** Bound filter. */ - private static class JsonBound extends JsonFilter { + @VisibleForTesting + protected static class JsonBound extends JsonFilter { private final String dimension; private final String lower; private final boolean lowerStrict; @@ -1206,9 +1227,13 @@ public class DruidQuery extends AbstractRelNode implements BindableRel { private final List<? extends JsonFilter> fields; private JsonCompositeFilter(String type, - List<? extends JsonFilter> fields) { + Iterable<? extends JsonFilter> fields) { super(type); - this.fields = fields; + this.fields = ImmutableList.copyOf(fields); + } + + private JsonCompositeFilter(String type, JsonFilter... fields) { + this(type, ImmutableList.copyOf(fields)); } public void write(JsonGenerator generator) throws IOException { @@ -1225,6 +1250,26 @@ public class DruidQuery extends AbstractRelNode implements BindableRel { } } + /** IN filter. */ + protected static class JsonInFilter extends JsonFilter { + private final String dimension; + private final List<String> values; + + private JsonInFilter(String type, String dimension, List<String> values) { + super(type); + this.dimension = dimension; + this.values = values; + } + + public void write(JsonGenerator generator) throws IOException { + generator.writeStartObject(); + generator.writeStringField("type", type); + generator.writeStringField("dimension", dimension); + writeField(generator, "values", values); + generator.writeEndObject(); + } + } + } // End DruidQuery.java http://git-wip-us.apache.org/repos/asf/calcite/blob/f8546915/druid/src/test/java/org/apache/calcite/adapter/druid/DruidQueryFilterTest.java ---------------------------------------------------------------------- diff --git a/druid/src/test/java/org/apache/calcite/adapter/druid/DruidQueryFilterTest.java b/druid/src/test/java/org/apache/calcite/adapter/druid/DruidQueryFilterTest.java new file mode 100644 index 0000000..2fc44de --- /dev/null +++ b/druid/src/test/java/org/apache/calcite/adapter/druid/DruidQueryFilterTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.druid; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeSystem; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.io.StringWriter; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.math.BigDecimal; +import java.util.List; + +import static org.hamcrest.core.Is.is; + +/** + * Tests generating Druid filters. + */ +public class DruidQueryFilterTest { + + @Test public void testInFilter() throws NoSuchMethodException, + InvocationTargetException, IllegalAccessException, IOException { + final Fixture f = new Fixture(); + final List<? extends RexNode> listRexNodes = + ImmutableList.of(f.rexBuilder.makeInputRef(f.varcharRowType, 0), + f.rexBuilder.makeExactLiteral(BigDecimal.valueOf(1)), + f.rexBuilder.makeExactLiteral(BigDecimal.valueOf(5)), + f.rexBuilder.makeLiteral("value1")); + + RexNode inRexNode = + f.rexBuilder.makeCall(SqlStdOperatorTable.IN, listRexNodes); + Method translateFilter = + DruidQuery.Translator.class.getDeclaredMethod("translateFilter", + RexNode.class); + translateFilter.setAccessible(true); + DruidQuery.JsonInFilter returnValue = + (DruidQuery.JsonInFilter) translateFilter.invoke(f.translatorStringKind, + inRexNode); + JsonFactory jsonFactory = new JsonFactory(); + final StringWriter sw = new StringWriter(); + JsonGenerator jsonGenerator = jsonFactory.createGenerator(sw); + returnValue.write(jsonGenerator); + jsonGenerator.close(); + + Assert.assertThat(sw.toString(), + is("{\"type\":\"in\",\"dimension\":\"dimensionName\"," + + "\"values\":[\"1\",\"5\",\"value1\"]}")); + } + + @Test public void testBetweenFilterStringCase() throws NoSuchMethodException, + InvocationTargetException, IllegalAccessException, IOException { + final Fixture f = new Fixture(); + final List<RexNode> listRexNodes = + ImmutableList.of(f.rexBuilder.makeLiteral(false), + f.rexBuilder.makeInputRef(f.varcharRowType, 0), + f.rexBuilder.makeLiteral("lower-bound"), + f.rexBuilder.makeLiteral("upper-bound")); + RelDataType relDataType = f.typeFactory.createSqlType(SqlTypeName.BOOLEAN); + RexNode betweenRexNode = f.rexBuilder.makeCall(relDataType, + SqlStdOperatorTable.BETWEEN, listRexNodes); + + Method translateFilter = + DruidQuery.Translator.class.getDeclaredMethod("translateFilter", + RexNode.class); + translateFilter.setAccessible(true); + DruidQuery.JsonBound returnValue = + (DruidQuery.JsonBound) translateFilter.invoke(f.translatorStringKind, + betweenRexNode); + JsonFactory jsonFactory = new JsonFactory(); + final StringWriter sw = new StringWriter(); + JsonGenerator jsonGenerator = jsonFactory.createGenerator(sw); + returnValue.write(jsonGenerator); + jsonGenerator.close(); + Assert.assertThat(sw.toString(), + is("{\"type\":\"bound\",\"dimension\":\"dimensionName\",\"lower\":\"lower-bound\"," + + "\"lowerStrict\":false,\"upper\":\"upper-bound\",\"upperStrict\":false," + + "\"alphaNumeric\":false}")); + } + + /** Everything a test needs for a healthy, active life. */ + static class Fixture { + final JavaTypeFactoryImpl typeFactory = + new JavaTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + final RexBuilder rexBuilder = new RexBuilder(typeFactory); + final DruidTable druidTable = + new DruidTable(Mockito.mock(DruidSchema.class), "dataSource", null, + ImmutableSet.<String>of(), "timestamp", null); + final RelDataType varcharType = + typeFactory.createSqlType(SqlTypeName.VARCHAR); + final RelDataType varcharRowType = typeFactory.builder() + .add("dimensionName", varcharType) + .build(); + final DruidQuery.Translator translatorStringKind = + new DruidQuery.Translator(druidTable, varcharRowType); + } +} + +// End DruidQueryFilterTest.java http://git-wip-us.apache.org/repos/asf/calcite/blob/f8546915/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 2340596..91b275b 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ limitations under the License. <maven-shade-plugin.version>2.1</maven-shade-plugin.version> <!-- Apache 18 has 3.0.0, but need 3.0.1 for [MSOURCES-94]. --> <maven-source-plugin.version>3.0.1</maven-source-plugin.version> - <mockito-all.version>1.10.19</mockito-all.version> + <mockito.version>2.5.5</mockito.version> <mongo-java-driver.version>2.12.3</mongo-java-driver.version> <mysql-driver.version>5.1.20</mysql-driver.version> <natty.version>0.13</natty.version> @@ -352,8 +352,8 @@ limitations under the License. </dependency> <dependency> <groupId>org.mockito</groupId> - <artifactId>mockito-all</artifactId> - <version>${mockito-all.version}</version> + <artifactId>mockito-core</artifactId> + <version>${mockito.version}</version> </dependency> <dependency> <groupId>org.postgresql</groupId>
