Repository: calcite Updated Branches: refs/heads/master 01be1ce69 -> 82cdc2fa7
[CALCITE-2094] Druid adapter: Count(*) returns null instead of 0 when condition filters all rows [CALCITE-2095] Druid adapter: Push always true and always true expressions as Expression Filters [CALCITE-2096] Druid adapter: Remove extra dummy_aggregator Close apache/calcite#584 Project: http://git-wip-us.apache.org/repos/asf/calcite/repo Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/82cdc2fa Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/82cdc2fa Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/82cdc2fa Branch: refs/heads/master Commit: 82cdc2fa7dd84e243f736a9772a03b17bef6215a Parents: 01be1ce Author: Slim <[email protected]> Authored: Sun Dec 17 19:13:03 2017 -0800 Committer: Jesus Camacho Rodriguez <[email protected]> Committed: Tue Dec 19 15:08:48 2017 -0800 ---------------------------------------------------------------------- .../calcite/adapter/druid/DruidQuery.java | 51 +++++++++++++++--- .../calcite/adapter/druid/DruidRules.java | 9 ---- .../org/apache/calcite/test/DruidAdapterIT.java | 57 ++++++++++---------- 3 files changed, 72 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite/blob/82cdc2fa/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java ---------------------------------------------------------------------- diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java index 3a7f25a..8a6524c 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java @@ -724,12 +724,6 @@ public class DruidQuery extends AbstractRelNode implements BindableRel { try { final JsonGenerator generator = factory.createGenerator(sw); - if (aggregations.isEmpty()) { - // Druid requires at least one aggregation, otherwise gives: - // Must have at least one AggregatorFactory - aggregations.add( - new JsonAggregation("longSum", "dummy_agg", "dummy_agg")); - } switch (queryType) { case TIMESERIES: generator.writeStartObject(); @@ -747,7 +741,11 @@ public class DruidQuery extends AbstractRelNode implements BindableRel { generator.writeFieldName("context"); // The following field is necessary to conform with SQL semantics (CALCITE-1589) generator.writeStartObject(); - generator.writeBooleanField("skipEmptyBuckets", true); + final boolean isCountStar = Granularity.ALL == finalGranularity + && aggregations.size() == 1 + && aggregations.get(0).type.equals("count"); + //Count(*) returns 0 if result set is empty thus need to set skipEmptyBuckets to false + generator.writeBooleanField("skipEmptyBuckets", !isCountStar); generator.writeEndObject(); generator.writeEndObject(); @@ -1146,6 +1144,12 @@ public class DruidQuery extends AbstractRelNode implements BindableRel { private JsonFilter translateFilter(RexNode e) { final RexCall call; + if (e.isAlwaysTrue()) { + return JsonExpressionFilter.alwaysTrue(); + } + if (e.isAlwaysFalse()) { + return JsonExpressionFilter.alwaysFalse(); + } switch (e.getKind()) { case EQUALS: case NOT_EQUALS: @@ -1459,7 +1463,8 @@ public class DruidQuery extends AbstractRelNode implements BindableRel { NOT, SELECTOR, IN, - BOUND; + BOUND, + EXPRESSION; public String lowercase() { return name().toLowerCase(Locale.ROOT); @@ -1473,6 +1478,36 @@ public class DruidQuery extends AbstractRelNode implements BindableRel { } } + /** + * Druid Expression filter. + */ + private static class JsonExpressionFilter extends JsonFilter { + private final String expression; + + private JsonExpressionFilter(String expression) { + super(Type.EXPRESSION); + this.expression = Preconditions.checkNotNull(expression); + } + + @Override public void write(JsonGenerator generator) throws IOException { + generator.writeStartObject(); + generator.writeStringField("type", type.lowercase()); + generator.writeStringField("expression", expression); + generator.writeEndObject(); + } + + /** We need to push to Druid an expression that always evaluates to true. */ + public static final JsonExpressionFilter alwaysTrue() { + return new JsonExpressionFilter("1 == 1"); + } + + /** We need to push to Druid an expression that always evaluates to false. */ + public static final JsonExpressionFilter alwaysFalse() { + return new JsonExpressionFilter("1 == 2"); + } + } + + /** Equality filter. */ private static class JsonSelector extends JsonFilter { private final String dimension; http://git-wip-us.apache.org/repos/asf/calcite/blob/82cdc2fa/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java ---------------------------------------------------------------------- diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java index 7e757e2..128f1aa 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java @@ -218,9 +218,6 @@ public class DruidRules { final RexSimplify simplify = new RexSimplify(rexBuilder, predicates, true, executor); final RexNode cond = simplify.simplify(filter.getCondition()); - if (!canPush(cond)) { - return; - } for (RexNode e : RelOptUtil.conjunctions(cond)) { if (query.isValidFilter(e)) { validPreds.add(e); @@ -317,12 +314,6 @@ public class DruidRules { } return ImmutableTriple.of(timeRangeNodes, pushableNodes, nonPushableNodes); } - - /** Returns whether we can push an expression to Druid. */ - private static boolean canPush(RexNode cond) { - // Druid cannot implement "where false" - return !cond.isAlwaysFalse(); - } } /** http://git-wip-us.apache.org/repos/asf/calcite/blob/82cdc2fa/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java ---------------------------------------------------------------------- diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java index d2a8755..2df8e57 100644 --- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java +++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java @@ -200,7 +200,7 @@ public class DruidAdapterIT { + "'dataSource':'wikiticker','granularity':'all'," + "'dimensions':[{'type':'default','dimension':'countryName'}],'limitSpec':{'type':'default'}," + "'filter':{'type':'selector','dimension':'page','value':'Jeremy Corbyn'}," - + "'aggregations':[{'type':'longSum','name':'dummy_agg','fieldName':'dummy_agg'}]," + + "'aggregations':[]," + "'intervals':['1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z']}"; sql(sql, WIKI_AUTO2) .returnsUnordered("countryName=United Kingdom", @@ -321,7 +321,7 @@ public class DruidAdapterIT { + "'dataSource':'wikiticker','granularity':'all'," + "'dimensions':[{'type':'default','dimension':'countryName'}],'limitSpec':{'type':'default'}," + "'filter':{'type':'selector','dimension':'page','value':'Jeremy Corbyn'}," - + "'aggregations':[{'type':'longSum','name':'dummy_agg','fieldName':'dummy_agg'}]," + + "'aggregations':[]," + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']}"; return sql(sql, url) .returnsUnordered("countryName=United Kingdom", @@ -413,7 +413,7 @@ public class DruidAdapterIT { final String sql = "select distinct \"state_province\" from \"foodmart\""; final String druidQuery = "{'queryType':'groupBy','dataSource':'foodmart','granularity':'all'," + "'dimensions':[{'type':'default','dimension':'state_province'}],'limitSpec':{'type':'default'}," - + "'aggregations':[{'type':'longSum','name':'dummy_agg','fieldName':'dummy_agg'}]," + + "'aggregations':[]," + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']}"; sql(sql) .returnsUnordered("state_province=CA", @@ -469,8 +469,7 @@ public class DruidAdapterIT { + "'dimension':'product_id'}]," + "'limitSpec':{'type':'default'},'filter':{'type':'selector'," + "'dimension':'product_id','value':'1020'}," - + "'aggregations':[{'type':'longSum','name':'dummy_agg'," - + "'fieldName':'dummy_agg'}]," + + "'aggregations':[]," + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']}"; sql(sql).queryContains(druidChecker(druidQuery)).returnsUnordered("product_id=1020"); } @@ -484,7 +483,7 @@ public class DruidAdapterIT { + "'dimensions':[{'type':'default','dimension':'product_id'}]," + "'limitSpec':{'type':'default'}," + "'filter':{'type':'selector','dimension':'product_id','value':'1020'}," - + "'aggregations':[{'type':'longSum','name':'dummy_agg','fieldName':'dummy_agg'}]," + + "'aggregations':[]," + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']}"; sql(sql) .returnsUnordered("id=1020") @@ -536,8 +535,7 @@ public class DruidAdapterIT { + "'columns':[{'dimension':'state_province','direction':'ascending'," + "'dimensionOrder':'alphanumeric'},{'dimension':'gender'," + "'direction':'descending','dimensionOrder':'alphanumeric'}]}," - + "'aggregations':[{'type':'longSum','name':'dummy_agg'," - + "'fieldName':'dummy_agg'}]," + + "'aggregations':[]," + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']}")) .explainContains(explain); } @@ -592,7 +590,7 @@ public class DruidAdapterIT { + "'granularity':'all','dimensions':[{'type':'default','dimension':'gender'}," + "{'type':'default','dimension':'state_province'}],'limitSpec':{'type':'default'," + "'limit':3,'columns':[]}," - + "'aggregations':[{'type':'longSum','name':'dummy_agg','fieldName':'dummy_agg'}]," + + "'aggregations':[]," + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']}"; final String explain = "PLAN=EnumerableInterpreter\n" + " DruidQuery(table=[[foodmart, foodmart]], " @@ -930,7 +928,7 @@ public class DruidAdapterIT { + "'descending':false,'granularity':'all'," + "'aggregations':[{'type':'count','name':'EXPR$0'}]," + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']," - + "'context':{'skipEmptyBuckets':true}}"; + + "'context':{'skipEmptyBuckets':false}}"; final String explain = "PLAN=EnumerableInterpreter\n" + " DruidQuery(table=[[foodmart, foodmart]], intervals=[[1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z]], projects=[[]], groups=[{}], aggs=[[COUNT()]])"; final String sql = "select count(*) from \"foodmart\""; @@ -1249,8 +1247,7 @@ public class DruidAdapterIT { final String druidQuery = "{'queryType':'groupBy','dataSource':'foodmart'," + "'granularity':'all','dimensions':[{'type':'default','dimension':'city'}," + "{'type':'default','dimension':'state_province'}]," - + "'limitSpec':{'type':'default'},'aggregations':[{'type':'longSum'," - + "'name':'dummy_agg','fieldName':'dummy_agg'}]," + + "'limitSpec':{'type':'default'},'aggregations':[]," + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']}"; sql(sql) .explainContains(explain) @@ -1293,7 +1290,7 @@ public class DruidAdapterIT { + "'value':'High Top Dried Mushrooms'},{'type':'or','fields':[{'type':'selector'," + "'dimension':'quarter','value':'Q2'},{'type':'selector','dimension':'quarter'," + "'value':'Q3'}]},{'type':'selector','dimension':'state_province','value':'WA'}]}," - + "'aggregations':[{'type':'longSum','name':'dummy_agg','fieldName':'dummy_agg'}]," + + "'aggregations':[]," + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']}"; final String explain = "PLAN=EnumerableInterpreter\n" + " DruidQuery(table=[[foodmart, foodmart]], intervals=[[1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z]]," @@ -1705,8 +1702,7 @@ public class DruidAdapterIT { + "'extractionFn':{'type':'timeFormat','format':'d','timeZone':'UTC'," + "'locale':'en-US'}},{'type':'selector','dimension':'__time'," + "'value':'11','extractionFn':{'type':'timeFormat','format':'M'," - + "'timeZone':'UTC','locale':'en-US'}}]},'aggregations':[{'type':'longSum'," - + "'name':'dummy_agg','fieldName':'dummy_agg'}]," + + "'timeZone':'UTC','locale':'en-US'}}]},'aggregations':[]," + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']}")) .returnsUnordered("product_id=1549; EXPR$1=30; EXPR$2=11", "product_id=1553; EXPR$1=30; EXPR$2=11"); @@ -1741,8 +1737,7 @@ public class DruidAdapterIT { + "'format':'M','timeZone':'UTC','locale':'en-US'}},{'type':'selector'," + "'dimension':'__time','value':'1997','extractionFn':{'type':'timeFormat'," + "'format':'yyyy','timeZone':'UTC','locale':'en-US'}}]}," - + "'aggregations':[{'type':'longSum','name':'dummy_agg'," - + "'fieldName':'dummy_agg'}]," + + "'aggregations':[]," + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']}")) .returnsUnordered("product_id=1549; EXPR$1=30; EXPR$2=11; EXPR$3=1997", "product_id=1553; EXPR$1=30; EXPR$2=11; EXPR$3=1997"); @@ -1765,8 +1760,7 @@ public class DruidAdapterIT { + "'timeZone':'UTC','locale':'en-US'}},{'type':'bound'," + "'dimension':'__time','upper':'11','upperStrict':false," + "'ordering':'numeric','extractionFn':{'type':'timeFormat','format':'M'," - + "'timeZone':'UTC','locale':'en-US'}}]},'aggregations':[{'type':'longSum'," - + "'name':'dummy_agg','fieldName':'dummy_agg'}]," + + "'timeZone':'UTC','locale':'en-US'}}]},'aggregations':[]," + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']}"; sql(sqlQuery) .returnsUnordered("product_id=1558; EXPR$1=10", "product_id=1558; EXPR$1=11", @@ -1793,8 +1787,7 @@ public class DruidAdapterIT { + "'format':'M','timeZone':'UTC','locale':'en-US'}},{'type':'selector'," + "'dimension':'__time','value':'11','extractionFn':{'type':'timeFormat'," + "'format':'M','timeZone':'UTC','locale':'en-US'}}]}]}," - + "'aggregations':[{'type':'longSum','name':'dummy_agg'," - + "'fieldName':'dummy_agg'}]," + + "'aggregations':[]," + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']}")) .returnsUnordered("product_id=1558; EXPR$1=10", "product_id=1558; EXPR$1=11", "product_id=1559; EXPR$1=11"); @@ -2023,8 +2016,7 @@ public class DruidAdapterIT { + "'locale':'en-US'}},{'type':'selector','dimension':'__time'," + "'value':'11','extractionFn':{'type':'timeFormat','format':'w'," + "'timeZone':'UTC','locale':'en-US'}}]}]}," - + "'aggregations':[{'type':'longSum','name':'dummy_agg'," - + "'fieldName':'dummy_agg'}]," + + "'aggregations':[]," + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']}"; sql(sql).returnsOrdered("EXPR$0=10\nEXPR$0=11").queryContains(druidChecker(druidQuery)); } @@ -2064,7 +2056,15 @@ public class DruidAdapterIT { @Test public void testFalseFilter() { String sql = "Select count(*) as c from \"foodmart\" where false"; - sql(sql).returnsUnordered("C=0"); + sql(sql) + .queryContains( + druidChecker("\"filter\":{\"type\":\"expression\",\"expression\":\"1 == 2\"}")) + .returnsUnordered("C=0"); + } + + @Test public void testTrueFilter() { + String sql = "Select count(*) as c from \"foodmart\" where true"; + sql(sql).returnsUnordered("C=86829"); } @Test public void testFalseFilterCaseConjectionWithTrue() { @@ -2120,7 +2120,7 @@ public class DruidAdapterIT { }) // Should return one row, "c=0"; logged // [CALCITE-1775] "GROUP BY ()" on empty relation should return 1 row - .returnsUnordered() + .returnsUnordered("c=0") .queryContains(druidChecker("'queryType':'timeseries'")); } @@ -3296,10 +3296,11 @@ public class DruidAdapterIT { + "'filter':{'type':'selector','dimension':'product_id','value':null}," + "'aggregations':[{'type':'count','name':'C'}]," + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']," - + "'context':{'skipEmptyBuckets':true}}"; + + "'context':{'skipEmptyBuckets':false}}"; sql(sql, FOODMART) .queryContains(druidChecker(druidQuery)) - .returnsCount(0); + .returnsUnordered("C=0") + .returnsCount(1); } @Test public void testIsNotNull() { @@ -3311,7 +3312,7 @@ public class DruidAdapterIT { + "'filter':{'type':'not','field':{'type':'selector','dimension':'product_id','value':null}}," + "'aggregations':[{'type':'count','name':'C'}]," + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']," - + "'context':{'skipEmptyBuckets':true}}"; + + "'context':{'skipEmptyBuckets':false}}"; sql(sql, FOODMART) .queryContains(druidChecker(druidQuery)) .returnsUnordered("C=86829");
