Repository: calcite Updated Branches: refs/heads/master ce0514690 -> 79af1c9ba
http://git-wip-us.apache.org/repos/asf/calcite/blob/79af1c9b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/QueryBuilders.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/QueryBuilders.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/QueryBuilders.java index 2ec4a0d..b046cb5 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/QueryBuilders.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/QueryBuilders.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.core.JsonGenerator; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Locale; import java.util.Objects; /** @@ -30,6 +29,10 @@ import java.util.Objects; * high-level client dependency on core modules (like lucene, netty, XContent etc.) which * is not compatible between different major versions. * + * <p>The goal of ES adapter is to + * be compatible with any elastic version or even to connect to clusters with different + * versions simultaneously. + * * <p>Jackson API is used to generate ES query as JSON document. */ class QueryBuilders { @@ -57,6 +60,16 @@ class QueryBuilders { } /** + * A Query that matches documents containing a single character term. + * + * @param name The name of the field + * @param value The value of the term + */ + static TermQueryBuilder termQuery(String name, char value) { + return new TermQueryBuilder(name, value); + } + + /** * A Query that matches documents containing a term. * * @param name The name of the field @@ -107,6 +120,16 @@ class QueryBuilders { } /** + * A filer for a field based on several terms matching on any of them. + * + * @param name The field name + * @param values The terms + */ + static TermsQueryBuilder termsQuery(String name, Iterable<?> values) { + return new TermsQueryBuilder(name, values); + } + + /** * A Query that matches documents within an range of terms. * * @param name The field name @@ -153,6 +176,13 @@ class QueryBuilders { } /** + * A query that matches on all documents. + */ + static MatchAllQueryBuilder matchAll() { + return new MatchAllQueryBuilder(); + } + + /** * Base class to build ES queries */ abstract static class QueryBuilder { @@ -246,34 +276,49 @@ class QueryBuilders { generator.writeFieldName("term"); generator.writeStartObject(); generator.writeFieldName(fieldName); - writeScalar(generator, value); + writeObject(generator, value); + generator.writeEndObject(); + generator.writeEndObject(); + } + } + + /** + * A filter for a field based on several terms matching on any of them. + */ + private static class TermsQueryBuilder extends QueryBuilder { + private final String fieldName; + private final Iterable<?> values; + + private TermsQueryBuilder(final String fieldName, final Iterable<?> values) { + this.fieldName = Objects.requireNonNull(fieldName, "fieldName"); + this.values = Objects.requireNonNull(values, "values"); + } + + @Override void writeJson(final JsonGenerator generator) throws IOException { + generator.writeStartObject(); + generator.writeFieldName("terms"); + generator.writeStartObject(); + generator.writeFieldName(fieldName); + generator.writeStartArray(); + for (Object value: values) { + writeObject(generator, value); + } + generator.writeEndArray(); generator.writeEndObject(); generator.writeEndObject(); } } /** - * Write single simple (scalar) value (string, number, boolean or null) to json output. + * Write usually simple (scalar) value (string, number, boolean or null) to json output. + * In case of complex objects delegates to jackson serialization. * * @param generator api to generate JSON document * @param value JSON value to write * @throws IOException if can't write to output */ - private static void writeScalar(JsonGenerator generator, Object value) throws IOException { - if (value == null) { - generator.writeNull(); - } else if (value instanceof CharSequence) { - generator.writeString(Objects.toString(value)); - } else if (value instanceof Number) { - // write numbers as string - generator.writeNumber(value.toString()); - } else if (value instanceof Boolean) { - generator.writeBoolean((Boolean) value); - } else { - final String message = String.format(Locale.ROOT, "Unsupported type %s : %s", - value.getClass(), value); - throw new IllegalArgumentException(message); - } + private static void writeObject(JsonGenerator generator, Object value) throws IOException { + generator.writeObject(value); } /** @@ -340,13 +385,13 @@ class QueryBuilders { if (gt != null) { final String op = gte ? "gte" : "gt"; generator.writeFieldName(op); - writeScalar(generator, gt); + writeObject(generator, gt); } if (lt != null) { final String op = lte ? "lte" : "lt"; generator.writeFieldName(op); - writeScalar(generator, lt); + writeObject(generator, lt); } if (format != null) { @@ -418,6 +463,27 @@ class QueryBuilders { generator.writeEndObject(); } } + + /** + * A query that matches on all documents. + * <pre> + * { + * "match_all": {} + * } + * </pre> + */ + static class MatchAllQueryBuilder extends QueryBuilder { + + private MatchAllQueryBuilder() {} + + @Override void writeJson(final JsonGenerator generator) throws IOException { + generator.writeStartObject(); + generator.writeFieldName("match_all"); + generator.writeStartObject(); + generator.writeEndObject(); + generator.writeEndObject(); + } + } } // End QueryBuilders.java http://git-wip-us.apache.org/repos/asf/calcite/blob/79af1c9b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationTest.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationTest.java new file mode 100644 index 0000000..8115bc7 --- /dev/null +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationTest.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.elasticsearch; + +import org.apache.calcite.jdbc.CalciteConnection; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.impl.ViewTable; +import org.apache.calcite.schema.impl.ViewTableMacro; +import org.apache.calcite.test.CalciteAssert; +import org.apache.calcite.test.ElasticsearchChecker; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; + +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Testing Elastic Search aggregation transformations. + */ +public class AggregationTest { + + @ClassRule + public static final EmbeddedElasticsearchPolicy NODE = EmbeddedElasticsearchPolicy.create(); + + private static final String NAME = "aggs"; + + @BeforeClass + public static void setupInstance() throws Exception { + + final Map<String, String> mappings = ImmutableMap.of("cat1", "keyword", + "cat2", "keyword", "cat3", "keyword", + "val1", "long", "val2", "long"); + + NODE.createIndex(NAME, mappings); + + String doc1 = "{'cat1': 'a', 'cat2': 'g', 'val1': 1 }".replace('\'', '"'); + String doc2 = "{'cat2': 'g', 'cat3': 'y', 'val2': 5 }".replace('\'', '"'); + String doc3 = "{'cat1': 'b', 'cat2':'h', 'cat3': 'z', 'val1': 7, 'val2': '42'}" + .replace('\'', '"'); + + List<ObjectNode> docs = new ArrayList<>(); + for (String text: Arrays.asList(doc1, doc2, doc3)) { + docs.add((ObjectNode) NODE.mapper().readTree(text)); + } + + NODE.insertBulk(NAME, docs); + } + + private CalciteAssert.ConnectionFactory newConnectionFactory() { + return new CalciteAssert.ConnectionFactory() { + @Override public Connection createConnection() throws SQLException { + final Connection connection = DriverManager.getConnection("jdbc:calcite:lex=JAVA"); + final SchemaPlus root = connection.unwrap(CalciteConnection.class).getRootSchema(); + + root.add("elastic", new ElasticsearchSchema(NODE.restClient(), NODE.mapper(), NAME)); + + // add calcite view programmatically + final String viewSql = String.format(Locale.ROOT, + "select _MAP['cat1'] AS \"cat1\", " + + " _MAP['cat2'] AS \"cat2\", " + + " _MAP['cat3'] AS \"cat3\", " + + " _MAP['val1'] AS \"val1\", " + + " _MAP['val2'] AS \"val2\" " + + " from \"elastic\".\"%s\"", NAME); + + ViewTableMacro macro = ViewTable.viewMacro(root, viewSql, + Collections.singletonList("elastic"), Arrays.asList("elastic", "view"), false); + root.add("view", macro); + return connection; + } + }; + } + + @Test + public void countStar() { + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select count(*) from view") + .queryContains( + ElasticsearchChecker.elasticsearchChecker("_source:false, size:0")) + .returns("EXPR$0=3\n"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select count(*) from view where cat1 = 'a'") + .returns("EXPR$0=1\n"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select count(*) from view where cat1 in ('a', 'b')") + .returns("EXPR$0=2\n"); + } + + @Test + public void all() throws Exception { + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select count(*), sum(val1), sum(val2) from view") + .queryContains( + ElasticsearchChecker.elasticsearchChecker("_source:false, size:0", + "aggregations:{'EXPR$0.value_count.field': '_id'", + "'EXPR$1.sum.field': 'val1'", + "'EXPR$2.sum.field': 'val2'}")) + .returns("EXPR$0=3; EXPR$1=8.0; EXPR$2=47.0\n"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select min(val1), max(val2), count(*) from view") + .queryContains( + ElasticsearchChecker.elasticsearchChecker("_source:false, size:0", + "aggregations:{'EXPR$0.min.field': 'val1'", + "'EXPR$1.max.field': 'val2'", + "'EXPR$2.value_count.field': '_id'}")) + .returns("EXPR$0=1.0; EXPR$1=42.0; EXPR$2=3\n"); + } + + @Test + public void cat1() throws Exception { + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select cat1, sum(val1), sum(val2) from view group by cat1") + .returnsUnordered("cat1=null; EXPR$1=0.0; EXPR$2=5.0", + "cat1=a; EXPR$1=1.0; EXPR$2=0.0", + "cat1=b; EXPR$1=7.0; EXPR$2=42.0"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select cat1, count(*) from view group by cat1") + .returnsUnordered("cat1=null; EXPR$1=1", + "cat1=a; EXPR$1=1", + "cat1=b; EXPR$1=1"); + + // different order for agg functions + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select count(*), cat1 from view group by cat1") + .returnsUnordered("EXPR$0=1; cat1=a", + "EXPR$0=1; cat1=b", + "EXPR$0=1; cat1=null"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select cat1, count(*), sum(val1), sum(val2) from view group by cat1") + .returnsUnordered("cat1=a; EXPR$1=1; EXPR$2=1.0; EXPR$3=0.0", + "cat1=b; EXPR$1=1; EXPR$2=7.0; EXPR$3=42.0", + "cat1=null; EXPR$1=1; EXPR$2=0.0; EXPR$3=5.0"); + } + + @Test + public void cat2() throws Exception { + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select cat2, min(val1), max(val1), min(val2), max(val2) from view group by cat2") + .returnsUnordered("cat2=g; EXPR$1=1.0; EXPR$2=1.0; EXPR$3=5.0; EXPR$4=5.0", + "cat2=h; EXPR$1=7.0; EXPR$2=7.0; EXPR$3=42.0; EXPR$4=42.0"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select cat2, sum(val1), sum(val2) from view group by cat2") + .returnsUnordered("cat2=g; EXPR$1=1.0; EXPR$2=5.0", + "cat2=h; EXPR$1=7.0; EXPR$2=42.0"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select cat2, count(*) from view group by cat2") + .returnsUnordered("cat2=g; EXPR$1=2", + "cat2=h; EXPR$1=1"); + } + + @Test + public void cat1Cat2() throws Exception { + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select cat1, cat2, sum(val1), sum(val2) from view group by cat1, cat2") + .returnsUnordered("cat1=a; cat2=g; EXPR$2=1.0; EXPR$3=0.0", + "cat1=null; cat2=g; EXPR$2=0.0; EXPR$3=5.0", + "cat1=b; cat2=h; EXPR$2=7.0; EXPR$3=42.0"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select cat1, cat2, count(*) from view group by cat1, cat2") + .returnsUnordered("cat1=a; cat2=g; EXPR$2=1", + "cat1=null; cat2=g; EXPR$2=1", + "cat1=b; cat2=h; EXPR$2=1"); + } + + @Test + public void cat1Cat3() throws Exception { + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select cat1, cat3, sum(val1), sum(val2) from view group by cat1, cat3") + .returnsUnordered("cat1=a; cat3=null; EXPR$2=1.0; EXPR$3=0.0", + "cat1=null; cat3=y; EXPR$2=0.0; EXPR$3=5.0", + "cat1=b; cat3=z; EXPR$2=7.0; EXPR$3=42.0"); + } + + @Test + public void cat1Cat2Cat3() throws Exception { + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select cat1, cat2, cat3, count(*), sum(val1), sum(val2) from view " + + "group by cat1, cat2, cat3") + .returnsUnordered("cat1=a; cat2=g; cat3=null; EXPR$3=1; EXPR$4=1.0; EXPR$5=0.0", + "cat1=b; cat2=h; cat3=z; EXPR$3=1; EXPR$4=7.0; EXPR$5=42.0", + "cat1=null; cat2=g; cat3=y; EXPR$3=1; EXPR$4=0.0; EXPR$5=5.0"); + } +} + +// End AggregationTest.java http://git-wip-us.apache.org/repos/asf/calcite/blob/79af1c9b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/BooleanLogicTest.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/BooleanLogicTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/BooleanLogicTest.java index c3b416c..c461905 100644 --- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/BooleanLogicTest.java +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/BooleanLogicTest.java @@ -128,6 +128,7 @@ public class BooleanLogicTest { assertSingle("select * from view where num > 41"); assertSingle("select * from view where num > 0"); assertSingle("select * from view where (a = 'a' and b = 'b') or (num = 42 and c = 'c')"); + assertSingle("select * from view where c = 'c' and (a in ('a', 'b') or num in (41, 42))"); assertSingle("select * from view where (a = 'a' or b = 'b') or (num = 42 and c = 'c')"); assertSingle("select * from view where a = 'a' and (b = '0' or (b = 'b' and " + "(c = '0' or (c = 'c' and num = 42))))"); http://git-wip-us.apache.org/repos/asf/calcite/blob/79af1c9b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java index d523bba..3d21b03 100644 --- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java @@ -92,7 +92,7 @@ public class ElasticSearchAdapterTest { private CalciteAssert.ConnectionFactory newConnectionFactory() { return new CalciteAssert.ConnectionFactory() { @Override public Connection createConnection() throws SQLException { - final Connection connection = DriverManager.getConnection("jdbc:calcite:"); + final Connection connection = DriverManager.getConnection("jdbc:calcite:lex=JAVA"); final SchemaPlus root = connection.unwrap(CalciteConnection.class).getRootSchema(); root.add("elastic", new ElasticsearchSchema(NODE.restClient(), NODE.mapper(), ZIPS)); @@ -108,7 +108,7 @@ public class ElasticSearchAdapterTest { ViewTableMacro macro = ViewTable.viewMacro(root, viewSql, Collections.singletonList("elastic"), Arrays.asList("elastic", "view"), false); - root.add("ZIPS", macro); + root.add("zips", macro); return connection; } @@ -126,7 +126,7 @@ public class ElasticSearchAdapterTest { @Test public void view() { calciteAssert() - .query("select * from zips where \"city\" = 'BROOKLYN'") + .query("select * from zips where city = 'BROOKLYN'") .returns("city=BROOKLYN; longitude=-73.956985; latitude=40.646694; " + "pop=111396; state=NY; id=11226\n") .returnsCount(1); @@ -141,42 +141,53 @@ public class ElasticSearchAdapterTest { CalciteAssert.that() .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" where _MAP['Foo'] = '_MISSING_'") + .query("select * from elastic.zips where _MAP['Foo'] = '_MISSING_'") .returnsCount(0); } @Test - public void basic() throws Exception { + public void basic() { CalciteAssert.that() .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" where _MAP['city'] = 'BROOKLYN'") + // by default elastic returns max 10 records + .query("select * from elastic.zips") + .runs(); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select * from elastic.zips where _MAP['city'] = 'BROOKLYN'") .returnsCount(1); CalciteAssert.that() .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" where" + .query("select * from elastic.zips where" + " _MAP['city'] in ('BROOKLYN', 'WASHINGTON')") .returnsCount(2); // lower-case CalciteAssert.that() .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" where " + .query("select * from elastic.zips where " + "_MAP['city'] in ('brooklyn', 'Brooklyn', 'BROOK') ") .returnsCount(0); // missing field CalciteAssert.that() .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" where _MAP['CITY'] = 'BROOKLYN'") + .query("select * from elastic.zips where _MAP['CITY'] = 'BROOKLYN'") .returnsCount(0); // limit works CalciteAssert.that() .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" limit 42") + .query("select * from elastic.zips limit 42") .returnsCount(42); + // limit 0 + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select * from elastic.zips limit 0") + .returnsCount(0); } @Test public void testSort() { @@ -186,14 +197,14 @@ public class ElasticSearchAdapterTest { + " ElasticsearchTableScan(table=[[elastic, zips]])"; calciteAssert() - .query("select * from zips order by \"state\"") + .query("select * from zips order by state") .returnsCount(10) .explainContains(explain); } @Test public void testSortLimit() { - final String sql = "select \"state\", \"pop\" from zips\n" - + "order by \"state\", \"pop\" offset 2 rows fetch next 3 rows only"; + final String sql = "select state, pop from zips\n" + + "order by state, pop offset 2 rows fetch next 3 rows only"; calciteAssert() .query(sql) .returnsUnordered("state=AK; pop=32383", @@ -201,44 +212,75 @@ public class ElasticSearchAdapterTest { "state=AL; pop=43862") .queryContains( ElasticsearchChecker.elasticsearchChecker( - "\"_source\" : [\"state\", \"pop\"]", - "\"sort\": [ {\"state\": \"asc\"}, {\"pop\": \"asc\"}]", - "\"from\": 2", - "\"size\": 3")); + "'_source' : ['state', 'pop']", + "sort: [ {state: 'asc'}, {pop: 'asc'}]", + "from: 2", + "size: 3")); } - + /** + * Sort by multiple fields (in different direction: asc/desc) + */ + @Test public void sortAscDesc() { + final String sql = "select city, state, pop from zips\n" + + "order by pop desc, state asc, city desc limit 3"; + calciteAssert() + .query(sql) + .returnsOrdered("city=CHICAGO; state=IL; pop=112047", + "city=BROOKLYN; state=NY; pop=111396", + "city=NEW YORK; state=NY; pop=106564") + .queryContains( + ElasticsearchChecker.elasticsearchChecker( + "'_source':['city','state','pop']", + "sort:[{pop:'desc'}, {state:'asc'}, {city:'desc'}]", + "size:3")); + } @Test public void testOffsetLimit() { - final String sql = "select \"state\", \"id\" from zips\n" + final String sql = "select state, id from zips\n" + "offset 2 fetch next 3 rows only"; calciteAssert() .query(sql) .runs() + .returnsCount(3) .queryContains( ElasticsearchChecker.elasticsearchChecker( - "\"from\": 2", - "\"size\": 3", - "\"_source\" : [\"state\", \"id\"]")); + "_source : ['state', 'id']", + "from: 2", + "size: 3")); } @Test public void testLimit() { - final String sql = "select \"state\", \"id\" from zips\n" + final String sql = "select state, id from zips\n" + "fetch next 3 rows only"; calciteAssert() .query(sql) .runs() + .returnsCount(3) .queryContains( ElasticsearchChecker.elasticsearchChecker( - "\"size\": 3", - "\"_source\" : [\"state\", \"id\"]")); + "'_source':['state','id']", + "size:3")); + } + + @Test + public void limit2() { + final String sql = "select id from zips limit 5"; + calciteAssert() + .query(sql) + .runs() + .returnsCount(5) + .queryContains( + ElasticsearchChecker.elasticsearchChecker( + "'_source':['id']", + "size:5")); } @Test public void testFilterSort() { final String sql = "select * from zips\n" - + "where \"state\" = 'CA' and \"pop\" >= 94000\n" - + "order by \"state\", \"pop\""; + + "where state = 'CA' and pop >= 94000\n" + + "order by state, pop"; final String explain = "PLAN=ElasticsearchToEnumerableConverter\n" + " ElasticsearchSort(sort0=[$4], sort1=[$3], dir0=[ASC], dir1=[ASC])\n" + " ElasticsearchProject(city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'), 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT], pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n" @@ -253,24 +295,24 @@ public class ElasticSearchAdapterTest { "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177;" + " pop=99568; state=CA; id=90201") .queryContains( - ElasticsearchChecker.elasticsearchChecker("\"query\" : " - + "{\"constant_score\":{\"filter\":{\"bool\":" - + "{\"must\":[{\"term\":{\"state\":\"CA\"}}," - + "{\"range\":{\"pop\":{\"gte\":94000}}}]}}}}", - "\"script_fields\": {\"longitude\":{\"script\":\"params._source.loc[0]\"}, " - + "\"latitude\":{\"script\":\"params._source.loc[1]\"}, " - + "\"city\":{\"script\": \"params._source.city\"}, " - + "\"pop\":{\"script\": \"params._source.pop\"}, " - + "\"state\":{\"script\": \"params._source.state\"}, " - + "\"id\":{\"script\": \"params._source.id\"}}", - "\"sort\": [ {\"state\": \"asc\"}, {\"pop\": \"asc\"}]")) + ElasticsearchChecker.elasticsearchChecker("'query' : " + + "{'constant_score':{filter:{bool:" + + "{must:[{term:{state:'CA'}}," + + "{range:{pop:{gte:94000}}}]}}}}", + "'script_fields': {longitude:{script:'params._source.loc[0]'}, " + + "latitude:{script:'params._source.loc[1]'}, " + + "city:{script: 'params._source.city'}, " + + "pop:{script: 'params._source.pop'}, " + + "state:{script: 'params._source.state'}, " + + "id:{script: 'params._source.id'}}", + "sort: [ {state: 'asc'}, {pop: 'asc'}]")) .explainContains(explain); } @Test public void testFilterSortDesc() { final String sql = "select * from zips\n" - + "where \"pop\" BETWEEN 95000 AND 100000\n" - + "order by \"state\" desc, \"pop\""; + + "where pop BETWEEN 95000 AND 100000\n" + + "order by state desc, pop"; calciteAssert() .query(sql) .limit(4) @@ -279,41 +321,20 @@ public class ElasticSearchAdapterTest { "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177; pop=99568; state=CA; id=90201"); } - @Ignore("Known issue when predicate analyzer doesn't simplify the expression (a = 1 and a > 0) ") - @Test public void testFilterRedundant() { - // known issue when PredicateAnalyzer doesn't simplify expressions - // (a < 3 and and a > 0 and a = 1) equivalent to (a = 1) - final String sql = "select * from zips\n" - + "where \"state\" > 'CA' and \"state\" < 'AZ' and \"state\" = 'OK'"; - calciteAssert() - .query(sql) - .runs() - .queryContains( - ElasticsearchChecker.elasticsearchChecker("" - + "\"query\" : {\"constant_score\":{\"filter\":{\"bool\":" - + "{\"must\":[{\"term\":{\"state\":\"OK\"}}]}}}}", - "\"script_fields\": {\"longitude\":{\"script\":\"params._source.loc[0]\"}, " - + "\"latitude\":{\"script\":\"params._source.loc[1]\"}, " - + "\"city\":{\"script\": \"params._source.city\"}, " - + "\"pop\":{\"script\": \"params._source.pop\"}, \"state\":{\"script\": \"params._source.state\"}, " - + "\"id\":{\"script\": \"params._source.id\"}}" - )); - } - @Test public void testInPlan() { final String[] searches = { - "\"query\" : {\"constant_score\":{\"filter\":{\"bool\":{\"should\":" - + "[{\"term\":{\"pop\":96074}},{\"term\":{\"pop\":99568}}]}}}}", - "\"script_fields\": {\"longitude\":{\"script\":\"params._source.loc[0]\"}, " - + "\"latitude\":{\"script\":\"params._source.loc[1]\"}, " - + "\"city\":{\"script\": \"params._source.city\"}, " - + "\"pop\":{\"script\": \"params._source.pop\"}, " - + "\"state\":{\"script\": \"params._source.state\"}, " - + "\"id\":{\"script\": \"params._source.id\"}}" + "'query' : {'constant_score':{filter:{bool:{should:" + + "[{term:{pop:96074}},{term:{pop:99568}}]}}}}", + "'script_fields': {longitude:{script:'params._source.loc[0]'}, " + + "latitude:{script:'params._source.loc[1]'}, " + + "city:{script: 'params._source.city'}, " + + "pop:{script: 'params._source.pop'}, " + + "state:{script: 'params._source.state'}, " + + "id:{script: 'params._source.id'}}" }; calciteAssert() - .query("select * from zips where \"pop\" in (96074, 99568)") + .query("select * from zips where pop in (96074, 99568)") .returnsUnordered( "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177; pop=99568; state=CA; id=90201", "city=LOS ANGELES; longitude=-118.258189; latitude=34.007856; pop=96074; state=CA; id=90011" @@ -323,14 +344,14 @@ public class ElasticSearchAdapterTest { @Test public void testZips() { calciteAssert() - .query("select \"state\", \"city\" from zips") + .query("select state, city from zips") .returnsCount(10); } @Test public void testProject() { - final String sql = "select \"state\", \"city\", 0 as \"zero\"\n" + final String sql = "select state, city, 0 as zero\n" + "from zips\n" - + "order by \"state\", \"city\""; + + "order by state, city"; calciteAssert() .query(sql) @@ -338,11 +359,11 @@ public class ElasticSearchAdapterTest { .returnsUnordered("state=AK; city=ANCHORAGE; zero=0", "state=AK; city=FAIRBANKS; zero=0") .queryContains( - ElasticsearchChecker.elasticsearchChecker("\"script_fields\": " - + "{\"zero\":{\"script\": \"0\"}, " - + "\"state\":{\"script\": \"params._source.state\"}, " - + "\"city\":{\"script\": \"params._source.city\"}}", - "\"sort\": [ {\"state\": \"asc\"}, {\"city\": \"asc\"}]")); + ElasticsearchChecker.elasticsearchChecker("script_fields:" + + "{zero:{script:'0'}," + + "state:{script:'params._source.state'}," + + "city:{script:'params._source.city'}}", + "sort:[{state:'asc'},{city:'asc'}]")); } @Test public void testFilter() { @@ -350,8 +371,9 @@ public class ElasticSearchAdapterTest { + " ElasticsearchProject(state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n" + " ElasticsearchFilter(condition=[=(CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'CA')])\n" + " ElasticsearchTableScan(table=[[elastic, zips]])"; + calciteAssert() - .query("select \"state\", \"city\" from zips where \"state\" = 'CA'") + .query("select state, city from zips where state = 'CA'") .limit(3) .returnsUnordered("state=CA; city=BELL GARDENS", "state=CA; city=LOS ANGELES", @@ -361,17 +383,142 @@ public class ElasticSearchAdapterTest { @Test public void testFilterReversed() { calciteAssert() - .query("select \"state\", \"city\" from zips where 'WI' < \"state\" order by \"city\"") + .query("select state, city from zips where 'WI' < state order by city") .limit(2) .returnsUnordered("state=WV; city=BECKLEY", "state=WY; city=CHEYENNE"); calciteAssert() - .query("select \"state\", \"city\" from zips where \"state\" > 'WI' order by \"city\"") + .query("select state, city from zips where state > 'WI' order by city") .limit(2) .returnsUnordered("state=WV; city=BECKLEY", "state=WY; city=CHEYENNE"); } + @Test + public void agg1() { + calciteAssert() + .query("select count(*) from zips") + .queryContains( + ElasticsearchChecker.elasticsearchChecker("'_source':false", + "size:0")) + .returns("EXPR$0=149\n"); + + // check with limit (should still return correct result). + calciteAssert() + .query("select count(*) from zips limit 1") + .returns("EXPR$0=149\n"); + + calciteAssert() + .query("select count(*) as cnt from zips") + .queryContains( + ElasticsearchChecker.elasticsearchChecker("'_source':false", + "size:0")) + .returns("cnt=149\n"); + + calciteAssert() + .query("select min(pop), max(pop) from zips") + .queryContains( + ElasticsearchChecker.elasticsearchChecker("'_source':false", + "size:0", + "aggregations:{'EXPR$0':{min:{field:'pop'}},'EXPR$1':{max:" + + "{field:'pop'}}}")) + .returns("EXPR$0=21; EXPR$1=112047\n"); + + calciteAssert() + .query("select min(pop) as min1, max(pop) as max1 from zips") + .returns("min1=21; max1=112047\n"); + + calciteAssert() + .query("select count(*), max(pop), min(pop), sum(pop), avg(pop) from zips") + .returns("EXPR$0=149; EXPR$1=112047; EXPR$2=21; EXPR$3=7865489; EXPR$4=52788\n"); + } + + @Test + public void groupBy() { + // ascending + calciteAssert() + .query("select min(pop), max(pop), state from zips group by state " + + "order by state limit 3") + .queryContains( + ElasticsearchChecker.elasticsearchChecker("'_source':false", + "size:0", + "aggregations:{'g_state':{terms:{field:'state',missing:'__MISSING__',size:3," + + " order:{'_key':'asc'}}", + "aggregations:{'EXPR$0':{min:{field:'pop'}},'EXPR$1':{max:{field:'pop'}}}}}")) + .returnsOrdered("EXPR$0=23238; EXPR$1=32383; state=AK", + "EXPR$0=42124; EXPR$1=44165; state=AL", + "EXPR$0=37428; EXPR$1=53532; state=AR"); + + // just one aggregation function + calciteAssert() + .query("select min(pop), state from zips group by state" + + " order by state limit 3") + .queryContains( + ElasticsearchChecker.elasticsearchChecker("'_source':false", + "size:0", + "aggregations:{'g_state':{terms:{field:'state',missing:'__MISSING__'," + + "size:3, order:{'_key':'asc'}}", + "aggregations:{'EXPR$0':{min:{field:'pop'}} }}}")) + .returnsOrdered("EXPR$0=23238; state=AK", + "EXPR$0=42124; state=AL", + "EXPR$0=37428; state=AR"); + + // group by count + calciteAssert() + .query("select count(city), state from zips group by state " + + "order by state limit 3") + .queryContains( + ElasticsearchChecker.elasticsearchChecker("'_source':false", + "size:0", + "aggregations:{'g_state':{terms:{field:'state',missing:'__MISSING__'," + + " size:3, order:{'_key':'asc'}}", + "aggregations:{'EXPR$0':{'value_count':{field:'city'}} }}}")) + .returnsOrdered("EXPR$0=3; state=AK", + "EXPR$0=3; state=AL", + "EXPR$0=3; state=AR"); + + // descending + calciteAssert() + .query("select min(pop), max(pop), state from zips group by state " + + " order by state desc limit 3") + .queryContains( + ElasticsearchChecker.elasticsearchChecker("'_source':false", + "size:0", + "aggregations:{'g_state':{terms:{field:'state',missing:'__MISSING__'," + + "size:3, order:{'_key':'desc'}}", + "aggregations:{'EXPR$0':{min:{field:'pop'}},'EXPR$1':" + + "{max:{field:'pop'}}}}}")) + .returnsOrdered("EXPR$0=25968; EXPR$1=33107; state=WY", + "EXPR$0=45196; EXPR$1=70185; state=WV", + "EXPR$0=51008; EXPR$1=57187; state=WI"); + } + + /** + * Checks + * <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-cardinality-aggregation.html";>Cardinality</a> + * aggregation {@code approx_count_distinct} + */ + @Test + @Ignore + public void approximateCount() throws Exception { + // approx_count_distinct is converted into two aggregations. needs investigation + // ElasticsearchAggregate(group=[{1}], EXPR$0=[COUNT($0)])\r + // ElasticsearchAggregate(group=[{0, 1}])\r + calciteAssert() + .query("select approx_count_distinct(city), state from zips group by state " + + "order by state limit 3") + .queryContains( + ElasticsearchChecker.elasticsearchChecker("'_source':false", + "size:0", + "aggregations:{'g_state':{terms:{field:state, size:3, " + + "order:{'_key':'asc'}}", + "aggregations:{'EXPR$0':{cardinality:{field:city}} }}}")) + .returnsOrdered("EXPR$0=3; state=AK", + "EXPR$0=3; state=AL", + "EXPR$0=3; state=AR"); + + } + } // End ElasticSearchAdapterTest.java http://git-wip-us.apache.org/repos/asf/calcite/blob/79af1c9b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchJsonTest.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchJsonTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchJsonTest.java new file mode 100644 index 0000000..0f2bf0c --- /dev/null +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchJsonTest.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.elasticsearch; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.hasItems; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.core.IsCollectionContaining.hasItem; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * Testing correct parsing of JSON (elasticsearch) response. + */ +public class ElasticsearchJsonTest { + + private ObjectMapper mapper; + + @Before + public void setUp() throws Exception { + this.mapper = new ObjectMapper() + .configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true) + .configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); + } + + @Test + public void aggEmpty() throws Exception { + String json = "{}"; + + ElasticsearchJson.Aggregations a = mapper.readValue(json, ElasticsearchJson.Aggregations.class); + assertNotNull(a); + assertThat(a.asList().size(), is(0)); + assertThat(a.asMap().size(), is(0)); + } + + @Test + public void aggSingle1() throws Exception { + String json = "{agg1: {value: '111'}}"; + + ElasticsearchJson.Aggregations a = mapper.readValue(json, ElasticsearchJson.Aggregations.class); + assertNotNull(a); + assertEquals(1, a.asList().size()); + assertEquals(1, a.asMap().size()); + assertEquals("agg1", a.asList().get(0).getName()); + assertEquals("agg1", a.asMap().keySet().iterator().next()); + assertEquals("111", ((ElasticsearchJson.MultiValue) a.asList().get(0)).value()); + + List<Map<String, Object>> rows = new ArrayList<>(); + ElasticsearchJson.visitValueNodes(a, rows::add); + assertThat(rows.size(), is(1)); + assertThat(rows.get(0).get("agg1"), is("111")); + } + + @Test + public void aggMultiValues() throws Exception { + String json = "{ agg1: {min: 0, max: 2, avg: 2.33}}"; + ElasticsearchJson.Aggregations a = mapper.readValue(json, ElasticsearchJson.Aggregations.class); + assertNotNull(a); + assertEquals(1, a.asList().size()); + assertEquals(1, a.asMap().size()); + assertEquals("agg1", a.asList().get(0).getName()); + + Map<String, Object> values = ((ElasticsearchJson.MultiValue) a.get("agg1")).values(); + assertThat(values.keySet(), hasItems("min", "max", "avg")); + } + + @Test + public void aggSingle2() throws Exception { + String json = "{ agg1: {value: 'foo'}, agg2: {value: 42}}"; + + ElasticsearchJson.Aggregations a = mapper.readValue(json, ElasticsearchJson.Aggregations.class); + assertNotNull(a); + assertEquals(2, a.asList().size()); + assertEquals(2, a.asMap().size()); + assertThat(a.asMap().keySet(), hasItems("agg1", "agg2")); + } + + @Test + public void aggBuckets1() throws Exception { + String json = "{ groupby: {buckets: [{key:'k1', doc_count:0, myagg:{value: 1.1}}," + + " {key:'k2', myagg:{value: 2.2}}] }}"; + + ElasticsearchJson.Aggregations a = mapper.readValue(json, ElasticsearchJson.Aggregations.class); + + assertThat(a.asMap().keySet(), hasItem("groupby")); + assertThat(a.get("groupby"), instanceOf(ElasticsearchJson.MultiBucketsAggregation.class)); + ElasticsearchJson.MultiBucketsAggregation multi = a.get("groupby"); + assertThat(multi.buckets().size(), is(2)); + assertThat(multi.getName(), is("groupby")); + assertThat(multi.buckets().get(0).key(), is("k1")); + assertThat(multi.buckets().get(0).keyAsString(), is("k1")); + assertThat(multi.buckets().get(1).key(), is("k2")); + assertThat(multi.buckets().get(1).keyAsString(), is("k2")); + } + + @Test + public void aggManyAggregations() throws Exception { + String json = "{groupby:{buckets:[" + + "{key:'k1', a1:{value:1}, a2:{value:2}}," + + "{key:'k2', a1:{value:3}, a2:{value:4}}" + + "]}}"; + + ElasticsearchJson.Aggregations a = mapper.readValue(json, ElasticsearchJson.Aggregations.class); + ElasticsearchJson.MultiBucketsAggregation multi = a.get("groupby"); + + assertThat(multi.buckets().get(0).getAggregations().asMap().size(), is(2)); + assertThat(multi.buckets().get(0).getName(), is("groupby")); + assertThat(multi.buckets().get(0).key(), is("k1")); + assertThat(multi.buckets().get(0).getAggregations().asMap().keySet(), hasItems("a1", "a2")); + assertThat(multi.buckets().get(1).getAggregations().asMap().size(), is(2)); + assertThat(multi.buckets().get(1).getName(), is("groupby")); + assertThat(multi.buckets().get(1).key(), is("k2")); + assertThat(multi.buckets().get(1).getAggregations().asMap().keySet(), hasItems("a1", "a2")); + List<Map<String, Object>> rows = new ArrayList<>(); + ElasticsearchJson.visitValueNodes(a, rows::add); + assertThat(rows.size(), is(2)); + assertThat(rows.get(0).get("groupby"), is("k1")); + assertThat(rows.get(0).get("a1"), is(1)); + assertThat(rows.get(0).get("a2"), is(2)); + } + + @Test + public void aggMultiBuckets() throws Exception { + String json = "{col1: {buckets: [" + + "{col2: {doc_count:1, buckets:[{key:'k3', max:{value:41}}]}, key:'k1'}," + + "{col2: {buckets:[{key:'k4', max:{value:42}}], doc_count:1}, key:'k2'}" + + "]}}"; + + ElasticsearchJson.Aggregations a = mapper.readValue(json, ElasticsearchJson.Aggregations.class); + assertNotNull(a); + + assertThat(a.asMap().keySet(), hasItem("col1")); + assertThat(a.get("col1"), instanceOf(ElasticsearchJson.MultiBucketsAggregation.class)); + ElasticsearchJson.MultiBucketsAggregation m = a.get("col1"); + assertThat(m.getName(), is("col1")); + assertThat(m.buckets().size(), is(2)); + assertThat(m.buckets().get(0).key(), is("k1")); + assertThat(m.buckets().get(0).getName(), is("col1")); + assertThat(m.buckets().get(0).getAggregations().asMap().keySet(), hasItem("col2")); + assertThat(m.buckets().get(1).key(), is("k2")); + List<Map<String, Object>> rows = new ArrayList<>(); + ElasticsearchJson.visitValueNodes(a, rows::add); + assertThat(rows.size(), is(2)); + + assertThat(rows.get(0).keySet(), hasItems("col1", "col2", "max")); + assertThat(rows.get(0).get("col1"), is("k1")); + assertThat(rows.get(0).get("col2"), is("k3")); + assertThat(rows.get(0).get("max"), is(41)); + + assertThat(rows.get(1).keySet(), hasItems("col1", "col2", "max")); + assertThat(rows.get(1).get("col1"), is("k2")); + assertThat(rows.get(1).get("col2"), is("k4")); + assertThat(rows.get(1).get("max"), is(42)); + } + +} + +// End ElasticsearchJsonTest.java http://git-wip-us.apache.org/repos/asf/calcite/blob/79af1c9b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/EmbeddedElasticsearchPolicy.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/EmbeddedElasticsearchPolicy.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/EmbeddedElasticsearchPolicy.java index 5cdd82f..bac2e6f 100644 --- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/EmbeddedElasticsearchPolicy.java +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/EmbeddedElasticsearchPolicy.java @@ -93,7 +93,16 @@ class EmbeddedElasticsearchPolicy extends ExternalResource { } /** - * Creates index in elastic search given mapping. + * Creates index in elastic search given a mapping. Mapping can contain nested fields expressed + * as dots({@code .}). + * + * <p>Example + * <pre> + * {@code + * b.a: long + * b.b: keyword + * } + * </pre> * * @param index index of the index * @param mapping field and field type mapping @@ -103,21 +112,37 @@ class EmbeddedElasticsearchPolicy extends ExternalResource { Objects.requireNonNull(index, "index"); Objects.requireNonNull(mapping, "mapping"); - ObjectNode json = mapper().createObjectNode(); + ObjectNode mappings = mapper().createObjectNode(); + + ObjectNode properties = mappings.with("mappings").with(index).with("properties"); for (Map.Entry<String, String> entry: mapping.entrySet()) { - json.set(entry.getKey(), json.objectNode().put("type", entry.getValue())); + applyMapping(properties, entry.getKey(), entry.getValue()); } - json = (ObjectNode) json.objectNode().set("properties", json); - json = (ObjectNode) json.objectNode().set(index, json); - json = (ObjectNode) json.objectNode().set("mappings", json); - // create index and mapping - final HttpEntity entity = new StringEntity(mapper().writeValueAsString(json), + final HttpEntity entity = new StringEntity(mapper().writeValueAsString(mappings), ContentType.APPLICATION_JSON); restClient().performRequest("PUT", "/" + index, Collections.emptyMap(), entity); } + /** + * Creates nested mappings for an index. This function is called recursively for each level. + * + * @param parent current parent + * @param key field name + * @param type ES mapping type ({@code keyword}, {@code long} etc.) + */ + private static void applyMapping(ObjectNode parent, String key, String type) { + final int index = key.indexOf('.'); + if (index > -1) { + String prefix = key.substring(0, index); + String suffix = key.substring(index + 1, key.length()); + applyMapping(parent.with(prefix).with("properties"), suffix, type); + } else { + parent.with(key).put("type", type); + } + } + void insertDocument(String index, ObjectNode document) throws IOException { Objects.requireNonNull(index, "index"); Objects.requireNonNull(document, "document"); http://git-wip-us.apache.org/repos/asf/calcite/blob/79af1c9b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/Projection2Test.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/Projection2Test.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/Projection2Test.java new file mode 100644 index 0000000..20d4457 --- /dev/null +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/Projection2Test.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.elasticsearch; + +import org.apache.calcite.jdbc.CalciteConnection; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.impl.ViewTable; +import org.apache.calcite.schema.impl.ViewTableMacro; +import org.apache.calcite.test.CalciteAssert; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; + +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Locale; +import java.util.Map; + +/** + * Checks renaming of fields (also upper, lower cases) during projections + */ +public class Projection2Test { + + @ClassRule + public static final EmbeddedElasticsearchPolicy NODE = EmbeddedElasticsearchPolicy.create(); + + private static final String NAME = "nested"; + + @BeforeClass + public static void setupInstance() throws Exception { + + final Map<String, String> mappings = ImmutableMap.of("a", "long", + "b.a", "long", "b.b", "long", "b.c.a", "keyword"); + + NODE.createIndex(NAME, mappings); + + String doc = "{'a': 1, 'b':{'a': 2, 'b':'3', 'c':{'a': 'foo'}}}".replace('\'', '"'); + NODE.insertDocument(NAME, (ObjectNode) NODE.mapper().readTree(doc)); + } + + private CalciteAssert.ConnectionFactory newConnectionFactory() { + return new CalciteAssert.ConnectionFactory() { + @Override public Connection createConnection() throws SQLException { + final Connection connection = DriverManager.getConnection("jdbc:calcite:"); + final SchemaPlus root = connection.unwrap(CalciteConnection.class).getRootSchema(); + + root.add("elastic", new ElasticsearchSchema(NODE.restClient(), NODE.mapper(), NAME)); + + // add calcite view programmatically + final String viewSql = String.format(Locale.ROOT, + "select _MAP['a'] AS \"a\", " + + " _MAP['b.a'] AS \"b.a\", " + + " _MAP['b.b'] AS \"b.b\", " + + " _MAP['b.c.a'] AS \"b.c.a\" " + + " from \"elastic\".\"%s\"", NAME); + + ViewTableMacro macro = ViewTable.viewMacro(root, viewSql, + Collections.singletonList("elastic"), Arrays.asList("elastic", "view"), false); + root.add("VIEW", macro); + return connection; + } + }; + } + + @Test + public void projection() { + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select \"a\", \"b.a\", \"b.b\", \"b.c.a\" from view") + .returns("a=1; b.a=2; b.b=3; b.c.a=foo\n"); + } + + @Test + public void projection2() { + String sql = String.format(Locale.ROOT, "select _MAP['a'], _MAP['b.a'], _MAP['b.b'], " + + "_MAP['b.c.a'], _MAP['missing'], _MAP['b.missing'] from \"elastic\".\"%s\"", NAME); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query(sql) + .returns("EXPR$0=1; EXPR$1=2; EXPR$2=3; EXPR$3=foo; EXPR$4=null; EXPR$5=null\n"); + } + +} + +// End Projection2Test.java http://git-wip-us.apache.org/repos/asf/calcite/blob/79af1c9b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ProjectionTest.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ProjectionTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ProjectionTest.java index 9830036..c4f95af 100644 --- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ProjectionTest.java +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ProjectionTest.java @@ -69,10 +69,10 @@ public class ProjectionTest { // add calcite view programmatically final String viewSql = String.format(Locale.ROOT, - "select cast(_MAP['A'] AS varchar(2)) AS \"a\", " - + " cast(_MAP['b'] AS varchar(2)) AS \"b\", " - + " cast(_MAP['cCC'] AS varchar(2)) AS \"c\", " - + " cast(_MAP['DDd'] AS varchar(2)) AS \"d\" " + "select cast(_MAP['A'] AS varchar(2)) AS a," + + " cast(_MAP['b'] AS varchar(2)) AS b, " + + " cast(_MAP['cCC'] AS varchar(2)) AS c, " + + " cast(_MAP['DDd'] AS varchar(2)) AS d " + " from \"elastic\".\"%s\"", NAME); ViewTableMacro macro = ViewTable.viewMacro(root, viewSql, @@ -89,8 +89,35 @@ public class ProjectionTest { CalciteAssert.that() .with(newConnectionFactory()) .query("select * from view") - .returns("a=aa; b=bb; c=cc; d=dd\n"); + .returns("A=aa; B=bb; C=cc; D=dd\n"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select a, b, c, d from view") + .returns("A=aa; B=bb; C=cc; D=dd\n"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select d, c, b, a from view") + .returns("D=dd; C=cc; B=bb; A=aa\n"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select a from view") + .returns("A=aa\n"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select a, b from view") + .returns("A=aa; B=bb\n"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select b, a from view") + .returns("B=bb; A=aa\n"); + } + } // End ProjectionTest.java http://git-wip-us.apache.org/repos/asf/calcite/blob/79af1c9b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/QueryBuildersTest.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/QueryBuildersTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/QueryBuildersTest.java index 7f11715..3f3099a 100644 --- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/QueryBuildersTest.java +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/QueryBuildersTest.java @@ -23,6 +23,13 @@ import org.junit.Test; import java.io.IOException; import java.io.StringWriter; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import static org.junit.Assert.assertEquals; @@ -33,14 +40,24 @@ public class QueryBuildersTest { private final ObjectMapper mapper = new ObjectMapper(); + /** + * Test for simple scalar terms (boolean, int etc.) + * @throws Exception not expected + */ @Test public void term() throws Exception { assertEquals("{\"term\":{\"foo\":\"bar\"}}", toJson(QueryBuilders.termQuery("foo", "bar"))); + assertEquals("{\"term\":{\"bar\":\"foo\"}}", + toJson(QueryBuilders.termQuery("bar", "foo"))); + assertEquals("{\"term\":{\"foo\":\"A\"}}", + toJson(QueryBuilders.termQuery("foo", 'A'))); assertEquals("{\"term\":{\"foo\":true}}", toJson(QueryBuilders.termQuery("foo", true))); assertEquals("{\"term\":{\"foo\":false}}", toJson(QueryBuilders.termQuery("foo", false))); + assertEquals("{\"term\":{\"foo\":0}}", + toJson(QueryBuilders.termQuery("foo", (byte) 0))); assertEquals("{\"term\":{\"foo\":123}}", toJson(QueryBuilders.termQuery("foo", (long) 123))); assertEquals("{\"term\":{\"foo\":41}}", @@ -49,6 +66,46 @@ public class QueryBuildersTest { toJson(QueryBuilders.termQuery("foo", 42.42D))); assertEquals("{\"term\":{\"foo\":1.1}}", toJson(QueryBuilders.termQuery("foo", 1.1F))); + assertEquals("{\"term\":{\"foo\":1}}", + toJson(QueryBuilders.termQuery("foo", new BigDecimal(1)))); + assertEquals("{\"term\":{\"foo\":121}}", + toJson(QueryBuilders.termQuery("foo", new BigInteger("121")))); + assertEquals("{\"term\":{\"foo\":111}}", + toJson(QueryBuilders.termQuery("foo", new AtomicLong(111)))); + assertEquals("{\"term\":{\"foo\":222}}", + toJson(QueryBuilders.termQuery("foo", new AtomicInteger(222)))); + assertEquals("{\"term\":{\"foo\":true}}", + toJson(QueryBuilders.termQuery("foo", new AtomicBoolean(true)))); + } + + @Test + public void terms() throws Exception { + assertEquals("{\"terms\":{\"foo\":[]}}", + toJson(QueryBuilders.termsQuery("foo", Collections.emptyList()))); + + assertEquals("{\"terms\":{\"bar\":[]}}", + toJson(QueryBuilders.termsQuery("bar", Collections.emptySet()))); + + assertEquals("{\"terms\":{\"singleton\":[0]}}", + toJson(QueryBuilders.termsQuery("singleton", Collections.singleton(0)))); + + assertEquals("{\"terms\":{\"foo\":[true]}}", + toJson(QueryBuilders.termsQuery("foo", Collections.singleton(true)))); + + assertEquals("{\"terms\":{\"foo\":[\"bar\"]}}", + toJson(QueryBuilders.termsQuery("foo", Collections.singleton("bar")))); + + assertEquals("{\"terms\":{\"foo\":[\"bar\"]}}", + toJson(QueryBuilders.termsQuery("foo", Collections.singletonList("bar")))); + + assertEquals("{\"terms\":{\"foo\":[true,false]}}", + toJson(QueryBuilders.termsQuery("foo", Arrays.asList(true, false)))); + + assertEquals("{\"terms\":{\"foo\":[1,2,3]}}", + toJson(QueryBuilders.termsQuery("foo", Arrays.asList(1, 2, 3)))); + + assertEquals("{\"terms\":{\"foo\":[1.1,2.2,3.3]}}", + toJson(QueryBuilders.termsQuery("foo", Arrays.asList(1.1, 2.2, 3.3)))); } @Test @@ -109,6 +166,12 @@ public class QueryBuildersTest { toJson(QueryBuilders.rangeQuery("f").lt(1).lt(2).lte(3))); } + @Test + public void matchAll() throws IOException { + assertEquals("{\"match_all\":{}}", + toJson(QueryBuilders.matchAll())); + } + private String toJson(QueryBuilders.QueryBuilder builder) throws IOException { StringWriter writer = new StringWriter(); JsonGenerator gen = mapper.getFactory().createGenerator(writer); http://git-wip-us.apache.org/repos/asf/calcite/blob/79af1c9b/elasticsearch/src/test/java/org/apache/calcite/test/ElasticsearchChecker.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/test/java/org/apache/calcite/test/ElasticsearchChecker.java b/elasticsearch/src/test/java/org/apache/calcite/test/ElasticsearchChecker.java index 823d4f1..0fe5ebc 100644 --- a/elasticsearch/src/test/java/org/apache/calcite/test/ElasticsearchChecker.java +++ b/elasticsearch/src/test/java/org/apache/calcite/test/ElasticsearchChecker.java @@ -16,15 +16,33 @@ */ package org.apache.calcite.test; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; /** * Internal util methods for ElasticSearch tests */ public class ElasticsearchChecker { + private static final ObjectMapper MAPPER = new ObjectMapper() + .enable(SerializationFeature.INDENT_OUTPUT) + .enable(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES) // user-friendly settings to + .enable(JsonParser.Feature.ALLOW_SINGLE_QUOTES); // avoid too much quoting + private ElasticsearchChecker() {} @@ -35,13 +53,75 @@ public class ElasticsearchChecker { */ public static Consumer<List> elasticsearchChecker(final String... strings) { Objects.requireNonNull(strings, "strings"); - return actual -> { - Object[] actualArray = actual == null || actual.isEmpty() ? null - : ((List) actual.get(0)).toArray(); - CalciteAssert.assertArrayEqual("expected Elasticsearch query not found", strings, - actualArray); + return a -> { + ObjectNode actual = a == null || a.isEmpty() ? null + : ((ObjectNode) a.get(0)); + + actual = expandDots(actual); + try { + + String json = "{" + Arrays.stream(strings).collect(Collectors.joining(",")) + "}"; + ObjectNode expected = (ObjectNode) MAPPER.readTree(json); + expected = expandDots(expected); + + if (!expected.equals(actual)) { + assertEquals("expected and actual Elasticsearch queries do not match", + MAPPER.writeValueAsString(expected), + MAPPER.writeValueAsString(actual)); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } }; } + + /** + * Expands attributes with dots ({@code .}) into sub-nodes. + * Use for more friendly JSON format: + * + * <pre> + * {'a.b.c': 1} + * expanded to + * {a: { b: {c: 1}}}} + * </pre> + * @param parent current node + * @param <T> type of node (usually JsonNode). + * @return copy of existing node with field {@code a.b.c} expanded. + */ + @SuppressWarnings("unchecked") + private static <T extends JsonNode> T expandDots(T parent) { + Objects.requireNonNull(parent, "parent"); + + if (parent.isValueNode()) { + return parent.deepCopy(); + } + + // ArrayNode + if (parent.isArray()) { + ArrayNode arr = (ArrayNode) parent; + ArrayNode copy = arr.arrayNode(); + arr.elements().forEachRemaining(e -> copy.add(expandDots(e))); + return (T) copy; + } + + // ObjectNode + ObjectNode objectNode = (ObjectNode) parent; + final ObjectNode copy = objectNode.objectNode(); + objectNode.fields().forEachRemaining(e -> { + final String property = e.getKey(); + final JsonNode node = e.getValue(); + + final String[] names = property.split("\\."); + ObjectNode copy2 = copy; + for (int i = 0; i < names.length - 1; i++) { + copy2 = copy2.with(names[i]); + } + copy2.set(names[names.length - 1], expandDots(node)); + }); + + return (T) copy; + } + } // End ElasticsearchChecker.java
