Fix-up for [CALCITE-2345] and [CALCITE-2347] Terminology: Standardize on Elasticsearch, not Elastic or ElasticSearch.
Rename MongoDatabaseRule to MongoDatabasePolicy to avoid confusion with planner rule classes; similarly ElasticsearchPolicy. Use project standard indentation. Ensure that files end in newlines. Project: http://git-wip-us.apache.org/repos/asf/calcite/repo Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/6e8bb5a1 Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/6e8bb5a1 Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/6e8bb5a1 Branch: refs/heads/master Commit: 6e8bb5a16c9c8c0397b79f3e98c0134ab9a32fb3 Parents: 4f83546 Author: Julian Hyde <[email protected]> Authored: Thu Jun 14 22:06:27 2018 -0700 Committer: Julian Hyde <[email protected]> Committed: Mon Jun 25 21:27:04 2018 -0700 ---------------------------------------------------------------------- .../java/org/apache/calcite/util/UtilTest.java | 1 + .../org/apache/calcite/test/DruidAdapterIT.java | 4 +- .../apache/calcite/test/DruidAdapterIT2.java | 4 +- .../Elasticsearch2Enumerator.java | 2 +- .../ElasticSearch2AdapterTest.java | 397 --------- .../Elasticsearch2AdapterTest.java | 398 +++++++++ .../elasticsearch2/EmbeddedElasticNode.java | 154 ---- .../elasticsearch2/EmbeddedElasticRule.java | 108 --- .../EmbeddedElasticsearchNode.java | 154 ++++ .../EmbeddedElasticsearchPolicy.java | 102 +++ .../org/apache/calcite/test/ElasticChecker.java | 53 -- .../calcite/test/ElasticsearchChecker.java | 53 ++ .../src/test/resources/zips-mini.json | 2 +- .../ElasticSearch5AdapterTest.java | 401 --------- .../Elasticsearch5AdapterTest.java | 405 +++++++++ .../EmbeddedElasticsearchNode.java | 2 +- .../EmbeddedElasticsearchPolicy.java | 103 +++ .../calcite/test/ElasticsearchChecker.java | 53 ++ elasticsearch5/src/test/resources/log4j2.xml | 2 +- .../src/test/resources/zips-mini.json | 2 +- .../adapter/mongodb/MongoAdapterTest.java | 812 +++++++++---------- .../adapter/mongodb/MongoDatabasePolicy.java | 84 ++ .../adapter/mongodb/MongoDatabaseRule.java | 84 -- .../org/apache/calcite/test/MongoAdapterIT.java | 6 +- mongodb/src/test/resources/zips-mini.json | 2 +- 25 files changed, 1772 insertions(+), 1616 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite/blob/6e8bb5a1/core/src/test/java/org/apache/calcite/util/UtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/util/UtilTest.java b/core/src/test/java/org/apache/calcite/util/UtilTest.java index 96f810f..4560fd5 100644 --- a/core/src/test/java/org/apache/calcite/util/UtilTest.java +++ b/core/src/test/java/org/apache/calcite/util/UtilTest.java @@ -843,6 +843,7 @@ public class UtilTest { Locale.getDefault(), Locale.US, Locale.TRADITIONAL_CHINESE, + Locale.ROOT, }; for (Locale locale : locales) { assertEquals(locale, Util.parseLocale(locale.toString())); http://git-wip-us.apache.org/repos/asf/calcite/blob/6e8bb5a1/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java ---------------------------------------------------------------------- diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java index 2a57303..36940b8 100644 --- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java +++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java @@ -71,8 +71,8 @@ import static org.junit.Assert.assertTrue; * <li>push HAVING into "groupBy" query</li> * </ul> * - * These tests use "timestamp with local time zone" type for the - * Druid timestamp column, instead of "timestamp" type as + * <p>These tests use TIMESTAMP WITH LOCAL TIME ZONE type for the + * Druid timestamp column, instead of TIMESTAMP type as * {@link DruidAdapterIT2}. */ public class DruidAdapterIT { http://git-wip-us.apache.org/repos/asf/calcite/blob/6e8bb5a1/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT2.java ---------------------------------------------------------------------- diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT2.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT2.java index 64f26de..c673c7c 100644 --- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT2.java +++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT2.java @@ -71,8 +71,8 @@ import static org.junit.Assert.assertTrue; * <li>push HAVING into "groupBy" query</li> * </ul> * - * These tests use "timestamp" type for the Druid timestamp column, instead - * of "timestamp with local time zone" type as {@link DruidAdapterIT}. + * <p>These tests use TIMESTAMP type for the Druid timestamp column, instead + * of TIMESTAMP WITH LOCAL TIME ZONE type as {@link DruidAdapterIT}. */ public class DruidAdapterIT2 { /** URL of the "druid-foodmart" model. */ http://git-wip-us.apache.org/repos/asf/calcite/blob/6e8bb5a1/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java ---------------------------------------------------------------------- diff --git a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java index 8247957..62aa197 100644 --- a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java +++ b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java @@ -115,7 +115,7 @@ public class Elasticsearch2Enumerator implements Enumerator<Object> { objects[i] = convert(hit.field(name).getValue(), field.getValue()); } else { throw new IllegalStateException( - String.format(Locale.getDefault(), "No result for %s", field)); + String.format(Locale.ROOT, "No result for %s", field)); } } return objects; http://git-wip-us.apache.org/repos/asf/calcite/blob/6e8bb5a1/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java ---------------------------------------------------------------------- diff --git a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java deleted file mode 100644 index a7eff8f..0000000 --- a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java +++ /dev/null @@ -1,397 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.adapter.elasticsearch2; - -import org.apache.calcite.jdbc.CalciteConnection; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.schema.impl.ViewTable; -import org.apache.calcite.schema.impl.ViewTableMacro; -import org.apache.calcite.test.CalciteAssert; -import org.apache.calcite.test.ElasticChecker; - -import com.google.common.io.LineProcessor; -import com.google.common.io.Resources; - -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; - -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.Collections; -import java.util.Locale; - -/** - * Set of tests for ES adapter. Uses real instance via {@link EmbeddedElasticRule}. Document - * source is local {@code zips-mini.json} file (located in the classpath). - */ -public class ElasticSearch2AdapterTest { - - @ClassRule //init once for all tests - public static final EmbeddedElasticRule NODE = EmbeddedElasticRule.create(); - - private static final String ZIPS = "zips"; - - /** - * Used to create {@code zips} index and insert some data - * - * @throws Exception when couldn't create the instance - */ - @BeforeClass - public static void setupInstance() throws Exception { - // define mapping so fields are searchable (term query) - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() - .startObject("properties") - .startObject("city").field("type", "string") - .field("index", "not_analyzed").endObject() - .startObject("state").field("type", "string") - .field("index", "not_analyzed").endObject() - .startObject("pop").field("type", "long").endObject() - .endObject() - .endObject(); - - // create index - NODE.client().admin().indices() - .prepareCreate(ZIPS) - .addMapping(ZIPS, mapping) - .get(); - - BulkRequestBuilder bulk = NODE.client().prepareBulk().setRefresh(true); - - // load records from file - Resources.readLines(ElasticSearch2AdapterTest.class.getResource("/zips-mini.json"), - StandardCharsets.UTF_8, new LineProcessor<Void>() { - @Override public boolean processLine(String line) throws IOException { - line = line.replaceAll("_id", "id"); // _id is a reserved attribute in ES - bulk.add(NODE.client().prepareIndex(ZIPS, ZIPS).setSource(line)); - return true; - } - - @Override public Void getResult() { - return null; - } - }); - - if (bulk.numberOfActions() == 0) { - throw new IllegalStateException("No records to be indexed"); - } - - BulkResponse response = bulk.execute().get(); - - if (response.hasFailures()) { - throw new IllegalStateException( - String.format(Locale.getDefault(), "Failed to populate %s:\n%s", NODE.httpAddress(), - Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed) - .map(BulkItemResponse::getFailureMessage).findFirst().orElse("<unknown>"))); - } - - } - - private CalciteAssert.ConnectionFactory newConnectionFactory() { - return new CalciteAssert.ConnectionFactory() { - @Override public Connection createConnection() throws SQLException { - final Connection connection = DriverManager.getConnection("jdbc:calcite:"); - final SchemaPlus root = connection.unwrap(CalciteConnection.class).getRootSchema(); - - root.add("elastic", new Elasticsearch2Schema(NODE.client(), ZIPS)); - - // add calcite view programmatically - final String viewSql = "select cast(_MAP['city'] AS varchar(20)) AS \"city\", " - + " cast(_MAP['loc'][0] AS float) AS \"longitude\",\n" - + " cast(_MAP['loc'][1] AS float) AS \"latitude\",\n" - + " cast(_MAP['pop'] AS integer) AS \"pop\", " - + " cast(_MAP['state'] AS varchar(2)) AS \"state\", " - + " cast(_MAP['id'] AS varchar(5)) AS \"id\" " - + "from \"elastic\".\"zips\""; - - ViewTableMacro macro = ViewTable.viewMacro(root, viewSql, - Collections.singletonList("elastic"), Arrays.asList("elastic", "view"), false); - root.add("ZIPS", macro); - - return connection; - } - }; - } - - private CalciteAssert.AssertThat calciteAssert() { - return CalciteAssert.that() - .with(newConnectionFactory()); - } - - /** - * Tests using calcite view - */ - @Test - public void view() { - calciteAssert() - .query("select * from zips where \"city\" = 'BROOKLYN'") - .returns("city=BROOKLYN; longitude=-73.956985; latitude=40.646694; " - + "pop=111396; state=NY; id=11226\n") - .returnsCount(1); - } - - @Test - public void emptyResult() { - CalciteAssert.that() - .with(newConnectionFactory()) - .query("select * from zips limit 0") - .returnsCount(0); - - CalciteAssert.that() - .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" where _MAP['Foo'] = '_MISSING_'") - .returnsCount(0); - } - - @Test - public void basic() throws Exception { - CalciteAssert.that() - .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" where _MAP['city'] = 'BROOKLYN'") - .returnsCount(1); - - CalciteAssert.that() - .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" where" - + " _MAP['city'] in ('BROOKLYN', 'WASHINGTON')") - .returnsCount(2); - - // lower-case - CalciteAssert.that() - .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" where " - + "_MAP['city'] in ('brooklyn', 'Brooklyn', 'BROOK') ") - .returnsCount(0); - - // missing field - CalciteAssert.that() - .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" where _MAP['CITY'] = 'BROOKLYN'") - .returnsCount(0); - - // limit works - CalciteAssert.that() - .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" limit 42") - .returnsCount(42); - - } - - @Test public void testSort() { - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n" - + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n" - + " ElasticsearchProject(city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'), 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT], pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n" - + " ElasticsearchTableScan(table=[[elastic, zips]])"; - - calciteAssert() - .query("select * from zips order by \"state\"") - .returnsCount(10) - .explainContains(explain); - } - - @Test public void testSortLimit() { - final String sql = "select \"state\", \"id\" from zips\n" - + "order by \"state\", \"id\" offset 2 rows fetch next 3 rows only"; - calciteAssert() - .query(sql) - .returnsUnordered("state=AK; id=99801", - "state=AL; id=35215", - "state=AL; id=35401") - .queryContains( - ElasticChecker.elasticsearchChecker( - "\"_source\" : [\"state\", \"id\"]", - "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]", - "\"from\": 2", - "\"size\": 3")); - } - - - - @Test public void testOffsetLimit() { - final String sql = "select \"state\", \"id\" from zips\n" - + "offset 2 fetch next 3 rows only"; - calciteAssert() - .query(sql) - .runs() - .queryContains( - ElasticChecker.elasticsearchChecker( - "\"from\": 2", - "\"size\": 3", - "\"_source\" : [\"state\", \"id\"]")); - } - - @Test public void testLimit() { - final String sql = "select \"state\", \"id\" from zips\n" - + "fetch next 3 rows only"; - - calciteAssert() - .query(sql) - .runs() - .queryContains( - ElasticChecker.elasticsearchChecker( - "\"size\": 3", - "\"_source\" : [\"state\", \"id\"]")); - } - - @Test public void testFilterSort() { - final String sql = "select * from zips\n" - + "where \"state\" = 'CA' and \"id\" >= '70000'\n" - + "order by \"state\", \"id\""; - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n" - + " ElasticsearchSort(sort0=[$4], sort1=[$5], dir0=[ASC], dir1=[ASC])\n" - + " ElasticsearchProject(city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'), 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT], pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n" - + " ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'CA'), >=(CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", '70000'))])\n" - + " ElasticsearchTableScan(table=[[elastic, zips]])"; - calciteAssert() - .query(sql) - .returnsOrdered("city=LOS ANGELES; longitude=-118.258189; latitude=34.007856; " - + "pop=96074; state=CA; id=90011", - "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177; " - + "pop=99568; state=CA; id=90201", - "city=NORWALK; longitude=-118.081767; latitude=33.90564; " - + "pop=94188; state=CA; id=90650") - .queryContains( - ElasticChecker.elasticsearchChecker("\"query\" : " - + "{\"constant_score\":{\"filter\":{\"bool\":" - + "{\"must\":[{\"term\":{\"state\":\"CA\"}}," - + "{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}", - "\"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, " - + "\"latitude\":{\"script\":\"_source.loc[1]\"}, " - + "\"city\":{\"script\": \"_source.city\"}, " - + "\"pop\":{\"script\": \"_source.pop\"}, " - + "\"state\":{\"script\": \"_source.state\"}, " - + "\"id\":{\"script\": \"_source.id\"}}", - "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]")) - .explainContains(explain); - } - - @Test public void testFilterSortDesc() { - final String sql = "select * from zips\n" - + "where \"pop\" BETWEEN 95000 AND 100000\n" - + "order by \"state\" desc, \"pop\""; - calciteAssert() - .query(sql) - .limit(4) - .returnsOrdered( - "city=LOS ANGELES; longitude=-118.258189; latitude=34.007856; pop=96074; state=CA; id=90011", - "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177; pop=99568; state=CA; id=90201"); - } - - @Test public void testFilterRedundant() { - final String sql = "select * from zips\n" - + "where \"state\" > 'CA' and \"state\" < 'AZ' and \"state\" = 'OK'"; - calciteAssert() - .query(sql) - .runs() - .queryContains( - ElasticChecker.elasticsearchChecker("" - + "\"query\" : {\"constant_score\":{\"filter\":{\"bool\":" - + "{\"must\":[{\"term\":{\"state\":\"OK\"}}]}}}}", - "\"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, " - + "\"latitude\":{\"script\":\"_source.loc[1]\"}, " - + "\"city\":{\"script\": \"_source.city\"}, " - + "\"pop\":{\"script\": \"_source.pop\"}, \"state\":{\"script\": \"_source.state\"}, " - + "\"id\":{\"script\": \"_source.id\"}}" - )); - } - - @Test public void testInPlan() { - final String[] searches = { - "\"query\" : {\"constant_score\":{\"filter\":{\"bool\":{\"should\":" - + "[{\"bool\":{\"must\":[{\"term\":{\"pop\":96074}}]}},{\"bool\":{\"must\":[{\"term\":" - + "{\"pop\":99568}}]}}]}}}}", - "\"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, " - + "\"latitude\":{\"script\":\"_source.loc[1]\"}, " - + "\"city\":{\"script\": \"_source.city\"}, " - + "\"pop\":{\"script\": \"_source.pop\"}, " - + "\"state\":{\"script\": \"_source.state\"}, " - + "\"id\":{\"script\": \"_source.id\"}}" - }; - - calciteAssert() - .query("select * from zips where \"pop\" in (96074, 99568)") - .returnsUnordered( - "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177; pop=99568; state=CA; id=90201", - "city=LOS ANGELES; longitude=-118.258189; latitude=34.007856; pop=96074; state=CA; id=90011" - ) - .queryContains(ElasticChecker.elasticsearchChecker(searches)); - } - - @Test public void testZips() { - calciteAssert() - .query("select \"state\", \"city\" from zips") - .returnsCount(10); - } - - @Test public void testProject() { - final String sql = "select \"state\", \"city\", 0 as \"zero\"\n" - + "from zips\n" - + "order by \"state\", \"city\""; - - calciteAssert() - .query(sql) - .limit(2) - .returnsUnordered("state=AK; city=ANCHORAGE; zero=0", - "state=AK; city=FAIRBANKS; zero=0") - .queryContains( - ElasticChecker.elasticsearchChecker("\"script_fields\": " - + "{\"zero\":{\"script\": \"0\"}, " - + "\"state\":{\"script\": \"_source.state\"}, " - + "\"city\":{\"script\": \"_source.city\"}}", - "\"sort\": [ {\"state\": \"asc\"}, {\"city\": \"asc\"}]")); - } - - @Test public void testFilter() { - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n" - + " ElasticsearchProject(state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n" - + " ElasticsearchFilter(condition=[=(CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'CA')])\n" - + " ElasticsearchTableScan(table=[[elastic, zips]])"; - calciteAssert() - .query("select \"state\", \"city\" from zips where \"state\" = 'CA'") - .limit(3) - .returnsUnordered("state=CA; city=BELL GARDENS", - "state=CA; city=LOS ANGELES", - "state=CA; city=NORWALK") - .explainContains(explain); - } - - @Test public void testFilterReversed() { - calciteAssert() - .query("select \"state\", \"city\" from zips where 'WI' < \"state\" order by \"city\"") - .limit(2) - .returnsUnordered("state=WV; city=BECKLEY", - "state=WY; city=CHEYENNE"); - calciteAssert() - .query("select \"state\", \"city\" from zips where \"state\" > 'WI' order by \"city\"") - .limit(2) - .returnsUnordered("state=WV; city=BECKLEY", - "state=WY; city=CHEYENNE"); - } - -} - -// End ElasticSearch2AdapterTest.java http://git-wip-us.apache.org/repos/asf/calcite/blob/6e8bb5a1/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2AdapterTest.java ---------------------------------------------------------------------- diff --git a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2AdapterTest.java b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2AdapterTest.java new file mode 100644 index 0000000..8624ac7 --- /dev/null +++ b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2AdapterTest.java @@ -0,0 +1,398 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.elasticsearch2; + +import org.apache.calcite.jdbc.CalciteConnection; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.impl.ViewTable; +import org.apache.calcite.schema.impl.ViewTableMacro; +import org.apache.calcite.test.CalciteAssert; +import org.apache.calcite.test.ElasticsearchChecker; + +import com.google.common.io.LineProcessor; +import com.google.common.io.Resources; + +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Locale; + +/** + * Set of tests for the Elasticsearch 2 adapter. + * + * <p>Uses a real instance via {@link EmbeddedElasticsearchPolicy}. + * The document source is a local {@code zips-mini.json} file + * (located in the classpath). + */ +public class Elasticsearch2AdapterTest { + + @ClassRule // init once for all tests + public static final EmbeddedElasticsearchPolicy NODE = + EmbeddedElasticsearchPolicy.create(); + + private static final String ZIPS = "zips"; + + /** + * Used to create {@code zips} index and insert some data + * + * @throws Exception when couldn't create the instance + */ + @BeforeClass + public static void setupInstance() throws Exception { + // define mapping so fields are searchable (term query) + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() + .startObject("properties") + .startObject("city").field("type", "string") + .field("index", "not_analyzed").endObject() + .startObject("state").field("type", "string") + .field("index", "not_analyzed").endObject() + .startObject("pop").field("type", "long").endObject() + .endObject() + .endObject(); + + // create index + NODE.client().admin().indices() + .prepareCreate(ZIPS) + .addMapping(ZIPS, mapping) + .get(); + + BulkRequestBuilder bulk = NODE.client().prepareBulk().setRefresh(true); + + // load records from file + Resources.readLines(Elasticsearch2AdapterTest.class.getResource("/zips-mini.json"), + StandardCharsets.UTF_8, new LineProcessor<Void>() { + @Override public boolean processLine(String line) throws IOException { + line = line.replaceAll("_id", "id"); // _id is a reserved attribute in ES + bulk.add(NODE.client().prepareIndex(ZIPS, ZIPS).setSource(line)); + return true; + } + + @Override public Void getResult() { + return null; + } + }); + + if (bulk.numberOfActions() == 0) { + throw new IllegalStateException("No records to be indexed"); + } + + BulkResponse response = bulk.execute().get(); + + if (response.hasFailures()) { + throw new IllegalStateException( + String.format(Locale.getDefault(), "Failed to populate %s:\n%s", NODE.httpAddress(), + Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed) + .map(BulkItemResponse::getFailureMessage).findFirst().orElse("<unknown>"))); + } + + } + + private CalciteAssert.ConnectionFactory newConnectionFactory() { + return new CalciteAssert.ConnectionFactory() { + @Override public Connection createConnection() throws SQLException { + final Connection connection = DriverManager.getConnection("jdbc:calcite:"); + final SchemaPlus root = connection.unwrap(CalciteConnection.class).getRootSchema(); + + root.add("elastic", new Elasticsearch2Schema(NODE.client(), ZIPS)); + + // add calcite view programmatically + final String viewSql = "select cast(_MAP['city'] AS varchar(20)) AS \"city\", " + + " cast(_MAP['loc'][0] AS float) AS \"longitude\",\n" + + " cast(_MAP['loc'][1] AS float) AS \"latitude\",\n" + + " cast(_MAP['pop'] AS integer) AS \"pop\", " + + " cast(_MAP['state'] AS varchar(2)) AS \"state\", " + + " cast(_MAP['id'] AS varchar(5)) AS \"id\" " + + "from \"elastic\".\"zips\""; + + ViewTableMacro macro = ViewTable.viewMacro(root, viewSql, + Collections.singletonList("elastic"), Arrays.asList("elastic", "view"), false); + root.add("ZIPS", macro); + + return connection; + } + }; + } + + private CalciteAssert.AssertThat calciteAssert() { + return CalciteAssert.that() + .with(newConnectionFactory()); + } + + /** + * Tests using calcite view + */ + @Test + public void view() { + calciteAssert() + .query("select * from zips where \"city\" = 'BROOKLYN'") + .returns("city=BROOKLYN; longitude=-73.956985; latitude=40.646694; " + + "pop=111396; state=NY; id=11226\n") + .returnsCount(1); + } + + @Test + public void emptyResult() { + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select * from zips limit 0") + .returnsCount(0); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select * from \"elastic\".\"zips\" where _MAP['Foo'] = '_MISSING_'") + .returnsCount(0); + } + + @Test + public void basic() throws Exception { + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select * from \"elastic\".\"zips\" where _MAP['city'] = 'BROOKLYN'") + .returnsCount(1); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select * from \"elastic\".\"zips\" where" + + " _MAP['city'] in ('BROOKLYN', 'WASHINGTON')") + .returnsCount(2); + + // lower-case + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select * from \"elastic\".\"zips\" where " + + "_MAP['city'] in ('brooklyn', 'Brooklyn', 'BROOK') ") + .returnsCount(0); + + // missing field + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select * from \"elastic\".\"zips\" where _MAP['CITY'] = 'BROOKLYN'") + .returnsCount(0); + + // limit works + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select * from \"elastic\".\"zips\" limit 42") + .returnsCount(42); + + } + + @Test public void testSort() { + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n" + + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n" + + " ElasticsearchProject(city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'), 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT], pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n" + + " ElasticsearchTableScan(table=[[elastic, zips]])"; + + calciteAssert() + .query("select * from zips order by \"state\"") + .returnsCount(10) + .explainContains(explain); + } + + @Test public void testSortLimit() { + final String sql = "select \"state\", \"id\" from zips\n" + + "order by \"state\", \"id\" offset 2 rows fetch next 3 rows only"; + calciteAssert() + .query(sql) + .returnsUnordered("state=AK; id=99801", + "state=AL; id=35215", + "state=AL; id=35401") + .queryContains( + ElasticsearchChecker.elasticsearchChecker( + "\"_source\" : [\"state\", \"id\"]", + "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]", + "\"from\": 2", + "\"size\": 3")); + } + + + + @Test public void testOffsetLimit() { + final String sql = "select \"state\", \"id\" from zips\n" + + "offset 2 fetch next 3 rows only"; + calciteAssert() + .query(sql) + .runs() + .queryContains( + ElasticsearchChecker.elasticsearchChecker( + "\"from\": 2", + "\"size\": 3", + "\"_source\" : [\"state\", \"id\"]")); + } + + @Test public void testLimit() { + final String sql = "select \"state\", \"id\" from zips\n" + + "fetch next 3 rows only"; + + calciteAssert() + .query(sql) + .runs() + .queryContains( + ElasticsearchChecker.elasticsearchChecker( + "\"size\": 3", + "\"_source\" : [\"state\", \"id\"]")); + } + + @Test public void testFilterSort() { + final String sql = "select * from zips\n" + + "where \"state\" = 'CA' and \"id\" >= '70000'\n" + + "order by \"state\", \"id\""; + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n" + + " ElasticsearchSort(sort0=[$4], sort1=[$5], dir0=[ASC], dir1=[ASC])\n" + + " ElasticsearchProject(city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'), 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT], pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n" + + " ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'CA'), >=(CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", '70000'))])\n" + + " ElasticsearchTableScan(table=[[elastic, zips]])"; + calciteAssert() + .query(sql) + .returnsOrdered("city=LOS ANGELES; longitude=-118.258189; latitude=34.007856; " + + "pop=96074; state=CA; id=90011", + "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177; " + + "pop=99568; state=CA; id=90201", + "city=NORWALK; longitude=-118.081767; latitude=33.90564; " + + "pop=94188; state=CA; id=90650") + .queryContains( + ElasticsearchChecker.elasticsearchChecker("\"query\" : " + + "{\"constant_score\":{\"filter\":{\"bool\":" + + "{\"must\":[{\"term\":{\"state\":\"CA\"}}," + + "{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}", + "\"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, " + + "\"latitude\":{\"script\":\"_source.loc[1]\"}, " + + "\"city\":{\"script\": \"_source.city\"}, " + + "\"pop\":{\"script\": \"_source.pop\"}, " + + "\"state\":{\"script\": \"_source.state\"}, " + + "\"id\":{\"script\": \"_source.id\"}}", + "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]")) + .explainContains(explain); + } + + @Test public void testFilterSortDesc() { + final String sql = "select * from zips\n" + + "where \"pop\" BETWEEN 95000 AND 100000\n" + + "order by \"state\" desc, \"pop\""; + calciteAssert() + .query(sql) + .limit(4) + .returnsOrdered( + "city=LOS ANGELES; longitude=-118.258189; latitude=34.007856; pop=96074; state=CA; id=90011", + "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177; pop=99568; state=CA; id=90201"); + } + + @Test public void testFilterRedundant() { + final String sql = "select * from zips\n" + + "where \"state\" > 'CA' and \"state\" < 'AZ' and \"state\" = 'OK'"; + calciteAssert() + .query(sql) + .runs() + .queryContains( + ElasticsearchChecker.elasticsearchChecker("" + + "\"query\" : {\"constant_score\":{\"filter\":{\"bool\":" + + "{\"must\":[{\"term\":{\"state\":\"OK\"}}]}}}}", + "\"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, " + + "\"latitude\":{\"script\":\"_source.loc[1]\"}, " + + "\"city\":{\"script\": \"_source.city\"}, " + + "\"pop\":{\"script\": \"_source.pop\"}, \"state\":{\"script\": \"_source.state\"}, " + + "\"id\":{\"script\": \"_source.id\"}}")); + } + + @Test public void testInPlan() { + final String[] searches = { + "\"query\" : {\"constant_score\":{\"filter\":{\"bool\":{\"should\":" + + "[{\"bool\":{\"must\":[{\"term\":{\"pop\":96074}}]}},{\"bool\":{\"must\":[{\"term\":" + + "{\"pop\":99568}}]}}]}}}}", + "\"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, " + + "\"latitude\":{\"script\":\"_source.loc[1]\"}, " + + "\"city\":{\"script\": \"_source.city\"}, " + + "\"pop\":{\"script\": \"_source.pop\"}, " + + "\"state\":{\"script\": \"_source.state\"}, " + + "\"id\":{\"script\": \"_source.id\"}}" + }; + + calciteAssert() + .query("select * from zips where \"pop\" in (96074, 99568)") + .returnsUnordered( + "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177; pop=99568; state=CA; id=90201", + "city=LOS ANGELES; longitude=-118.258189; latitude=34.007856; pop=96074; state=CA; id=90011") + .queryContains(ElasticsearchChecker.elasticsearchChecker(searches)); + } + + @Test public void testZips() { + calciteAssert() + .query("select \"state\", \"city\" from zips") + .returnsCount(10); + } + + @Test public void testProject() { + final String sql = "select \"state\", \"city\", 0 as \"zero\"\n" + + "from zips\n" + + "order by \"state\", \"city\""; + + calciteAssert() + .query(sql) + .limit(2) + .returnsUnordered("state=AK; city=ANCHORAGE; zero=0", + "state=AK; city=FAIRBANKS; zero=0") + .queryContains( + ElasticsearchChecker.elasticsearchChecker("\"script_fields\": " + + "{\"zero\":{\"script\": \"0\"}, " + + "\"state\":{\"script\": \"_source.state\"}, " + + "\"city\":{\"script\": \"_source.city\"}}", + "\"sort\": [ {\"state\": \"asc\"}, {\"city\": \"asc\"}]")); + } + + @Test public void testFilter() { + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n" + + " ElasticsearchProject(state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n" + + " ElasticsearchFilter(condition=[=(CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'CA')])\n" + + " ElasticsearchTableScan(table=[[elastic, zips]])"; + calciteAssert() + .query("select \"state\", \"city\" from zips where \"state\" = 'CA'") + .limit(3) + .returnsUnordered("state=CA; city=BELL GARDENS", + "state=CA; city=LOS ANGELES", + "state=CA; city=NORWALK") + .explainContains(explain); + } + + @Test public void testFilterReversed() { + calciteAssert() + .query("select \"state\", \"city\" from zips where 'WI' < \"state\" order by \"city\"") + .limit(2) + .returnsUnordered("state=WV; city=BECKLEY", + "state=WY; city=CHEYENNE"); + calciteAssert() + .query("select \"state\", \"city\" from zips where \"state\" > 'WI' order by \"city\"") + .limit(2) + .returnsUnordered("state=WV; city=BECKLEY", + "state=WY; city=CHEYENNE"); + } + +} + +// End Elasticsearch2AdapterTest.java http://git-wip-us.apache.org/repos/asf/calcite/blob/6e8bb5a1/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java ---------------------------------------------------------------------- diff --git a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java deleted file mode 100644 index 55f83f9..0000000 --- a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.adapter.elasticsearch2; - -import com.google.common.base.Preconditions; -import com.google.common.io.Files; - -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.internal.InternalSettingsPreparer; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.groovy.GroovyPlugin; - -import java.io.File; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; - -/** - * Represents a single elastic search node which can run embedded in a java application. - * Intended for unit and integration tests. Settings and plugins are crafted for Calcite. - */ -class EmbeddedElasticNode implements AutoCloseable { - - private final LocalNode node; - private volatile boolean isStarted; - - private EmbeddedElasticNode(LocalNode node) { - this.node = Preconditions.checkNotNull(node, "node"); - } - - /** - * Having separate class to expose (protected) constructor which allows to install - * different plugins. In our case it is {@code GroovyPlugin} for scripted fields like - * {@code loc[0]} or {@code loc[1]['foo']}. - */ - private static class LocalNode extends Node { - private LocalNode(Settings settings, Collection<Class<? extends Plugin>> classpathPlugins) { - super(InternalSettingsPreparer.prepareEnvironment(settings, null), - Version.CURRENT, classpathPlugins); - } - } - - /** - * Creates an instance with existing settings - * - * @param settings ES settings for the node - * @return un-started node; call {@link #start()} to start the instance - */ - private static EmbeddedElasticNode create(Settings settings) { - // ensure GroovyPlugin is installed or otherwise scripted fields would not work - LocalNode node = new LocalNode(settings, Collections.singleton(GroovyPlugin.class)); - return new EmbeddedElasticNode(node); - } - - /** - * Creates elastic node as single member of a cluster. Node will not be started - * unless {@link #start()} is explicitly called. - * - * @return node with default configuration - */ - public static EmbeddedElasticNode create() { - File data = Files.createTempDir(); - data.deleteOnExit(); - File home = Files.createTempDir(); - home.deleteOnExit(); - - Settings settings = Settings.builder() - .put("node.name", "fake-elastic") - .put("path.home", home.getAbsolutePath()) - .put("path.data", data.getAbsolutePath()) - .put("script.inline", true) // requires GroovyPlugin - .put("script.indexed", true) // requires GroovyPlugin - .put("cluster.routing.allocation.disk.threshold_enabled", false) - .put("node.local", true) - .put("node.data", true) - .put("network.host", "localhost") - .build(); - - return create(settings); - } - - /** - * Starts current node - */ - public void start() { - Preconditions.checkState(!isStarted, "already started"); - node.start(); - this.isStarted = true; - } - - /** - * Returns the current address to connect to with HTTP client. - * - * @return {@code HTTP} address (hostname / port) - */ - public TransportAddress httpAddress() { - Preconditions.checkState(isStarted, "node is not started"); - - NodesInfoResponse response = client().admin().cluster().prepareNodesInfo() - .execute().actionGet(); - if (response.getNodes().length != 1) { - throw new IllegalStateException("Expected single node but got " - + response.getNodes().length); - } - NodeInfo node = response.getNodes()[0]; - return node.getHttp().address().boundAddresses()[0]; - } - - /** - * Exposes elastic - * <a href="https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/transport-client.html";>transport client</a> - * (use of HTTP client is preferred). - * - * @return client API to access ES functionality - */ - public Client client() { - Preconditions.checkState(isStarted, "node is not started"); - return node.client(); - } - - @Override public void close() throws Exception { - node.close(); - // cleanup data dirs - File data = new File(node.settings().get("path.data")); - File home = new File(node.settings().get("path.home")); - for (File file: Arrays.asList(data, home)) { - if (file.exists()) { - file.delete(); - } - } - } -} - -// End EmbeddedElasticNode.java http://git-wip-us.apache.org/repos/asf/calcite/blob/6e8bb5a1/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java ---------------------------------------------------------------------- diff --git a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java deleted file mode 100644 index 4bd9c25..0000000 --- a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.adapter.elasticsearch2; - -import com.google.common.base.Preconditions; - -import org.elasticsearch.client.Client; -import org.elasticsearch.common.transport.TransportAddress; -import org.junit.rules.ExternalResource; - -/** - * Used to initialize a single elastic node. For performance reasons (node startup costs), - * same instance is usually shared across multiple tests. - * - * This rule should be used as follows: - * <pre> - * - * public class MyTest { - * @ClassRule - * public static final ElasticSearchRule RULE = ElasticSearchRule.create(); - * - * @BeforeClass - * public static void setup() { - * // ... populate instance - * } - * - * @Test - * public void myTest() { - * TransportAddress address = RULE.httpAddress(); - * // .... (connect) - * } - * } - * </pre> - * - * @see ExternalResource - */ -class EmbeddedElasticRule extends ExternalResource { - - private final EmbeddedElasticsearchNode node; - - private EmbeddedElasticsearchPolicy(EmbeddedElasticsearchNode resource) { - this.node = Preconditions.checkNotNull(resource, "resource"); - } - - @Override protected void before() throws Throwable { - node.start(); - } - - @Override protected void after() { - try { - node.close(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** - * Factory method to create this rule. - * - * @return new rule instance to be used in unit tests - */ -<<<<<<< HEAD:elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java - public static EmbeddedElasticRule create() { - return new EmbeddedElasticRule(EmbeddedElasticNode.create()); -======= - public static EmbeddedElasticsearchPolicy create() { - return new EmbeddedElasticsearchPolicy(EmbeddedElasticsearchNode.create()); ->>>>>>> 54c598206... more 2380:elasticsearch5/src/test/java/org/apache/calcite/adapter/elasticsearch5/EmbeddedElasticsearchPolicy.java - } - - /** - * Exposes current ES transport client. - * - * @return initialized instance of ES - */ - Client client() { - return node.client(); - } - - /** - * HTTP address for rest clients (can be ES native or any other). - * -<<<<<<< HEAD:elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java - * @return HTTP hostname/port to connect to this ES instance -======= - * @return {@code HTTP} connection parameters ->>>>>>> 54c598206... more 2380:elasticsearch5/src/test/java/org/apache/calcite/adapter/elasticsearch5/EmbeddedElasticsearchPolicy.java - */ - TransportAddress httpAddress() { - return node.httpAddress(); - } -} - -// End EmbeddedElasticRule.java http://git-wip-us.apache.org/repos/asf/calcite/blob/6e8bb5a1/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticsearchNode.java ---------------------------------------------------------------------- diff --git a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticsearchNode.java b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticsearchNode.java new file mode 100644 index 0000000..d7e3464 --- /dev/null +++ b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticsearchNode.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.elasticsearch2; + +import com.google.common.base.Preconditions; +import com.google.common.io.Files; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.internal.InternalSettingsPreparer; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.groovy.GroovyPlugin; + +import java.io.File; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; + +/** + * Represents a single Elasticsearch node that can run embedded in a java application. + * Intended for unit and integration tests. Settings and plugins are crafted for Calcite. + */ +class EmbeddedElasticsearchNode implements AutoCloseable { + + private final LocalNode node; + private volatile boolean isStarted; + + private EmbeddedElasticsearchNode(LocalNode node) { + this.node = Preconditions.checkNotNull(node, "node"); + } + + /** + * Having separate class to expose (protected) constructor which allows to install + * different plugins. In our case it is {@code GroovyPlugin} for scripted fields like + * {@code loc[0]} or {@code loc[1]['foo']}. + */ + private static class LocalNode extends Node { + private LocalNode(Settings settings, Collection<Class<? extends Plugin>> classpathPlugins) { + super(InternalSettingsPreparer.prepareEnvironment(settings, null), + Version.CURRENT, classpathPlugins); + } + } + + /** + * Creates an instance with existing settings + * + * @param settings ES settings for the node + * @return un-started node; call {@link #start()} to start the instance + */ + private static EmbeddedElasticsearchNode create(Settings settings) { + // ensure GroovyPlugin is installed or otherwise scripted fields would not work + LocalNode node = new LocalNode(settings, Collections.singleton(GroovyPlugin.class)); + return new EmbeddedElasticsearchNode(node); + } + + /** + * Creates elastic node as single member of a cluster. Node will not be started + * unless {@link #start()} is explicitly called. + * + * @return node with default configuration + */ + public static EmbeddedElasticsearchNode create() { + File data = Files.createTempDir(); + data.deleteOnExit(); + File home = Files.createTempDir(); + home.deleteOnExit(); + + Settings settings = Settings.builder() + .put("node.name", "fake-elastic") + .put("path.home", home.getAbsolutePath()) + .put("path.data", data.getAbsolutePath()) + .put("script.inline", true) // requires GroovyPlugin + .put("script.indexed", true) // requires GroovyPlugin + .put("cluster.routing.allocation.disk.threshold_enabled", false) + .put("node.local", true) + .put("node.data", true) + .put("network.host", "localhost") + .build(); + + return create(settings); + } + + /** + * Starts current node + */ + public void start() { + Preconditions.checkState(!isStarted, "already started"); + node.start(); + this.isStarted = true; + } + + /** + * Returns the current address to connect to with HTTP client. + * + * @return {@code HTTP} address (hostname / port) + */ + public TransportAddress httpAddress() { + Preconditions.checkState(isStarted, "node is not started"); + + NodesInfoResponse response = client().admin().cluster().prepareNodesInfo() + .execute().actionGet(); + if (response.getNodes().length != 1) { + throw new IllegalStateException("Expected single node but got " + + response.getNodes().length); + } + NodeInfo node = response.getNodes()[0]; + return node.getHttp().address().boundAddresses()[0]; + } + + /** + * Exposes an Elasticsearch + * <a href="https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/transport-client.html";>transport client</a> + * (use of HTTP client is preferred). + * + * @return client API to access ES functionality + */ + public Client client() { + Preconditions.checkState(isStarted, "node is not started"); + return node.client(); + } + + @Override public void close() throws Exception { + node.close(); + // cleanup data dirs + File data = new File(node.settings().get("path.data")); + File home = new File(node.settings().get("path.home")); + for (File file: Arrays.asList(data, home)) { + if (file.exists()) { + file.delete(); + } + } + } +} + +// End EmbeddedElasticsearchNode.java http://git-wip-us.apache.org/repos/asf/calcite/blob/6e8bb5a1/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticsearchPolicy.java ---------------------------------------------------------------------- diff --git a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticsearchPolicy.java b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticsearchPolicy.java new file mode 100644 index 0000000..6b25ecf --- /dev/null +++ b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticsearchPolicy.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.elasticsearch2; + +import com.google.common.base.Preconditions; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.transport.TransportAddress; +import org.junit.rules.ExternalResource; + +/** + * Junit rule that is used to initialize a single Elasticsearch node for tests. + * + * <p>For performance reasons (node startup costs), + * the same instance is usually shared across multiple tests. + * + * <p>This rule should be used as follows: + * <pre> + * + * public class MyTest { + * @ClassRule + * public static final EmbeddedElasticsearchPolicy POLICY = + * EmbeddedElasticsearchPolicy.create(); + * + * @BeforeClass + * public static void setup() { + * // ... populate instance + * } + * + * @Test + * public void myTest() { + * TransportAddress address = POLICY.httpAddress(); + * // .... (connect) + * } + * } + * </pre> + * + * @see ExternalResource + */ +class EmbeddedElasticsearchPolicy extends ExternalResource { + + private final EmbeddedElasticsearchNode node; + + private EmbeddedElasticsearchPolicy(EmbeddedElasticsearchNode resource) { + this.node = Preconditions.checkNotNull(resource, "resource"); + } + + @Override protected void before() throws Throwable { + node.start(); + } + + @Override protected void after() { + try { + node.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Factory method to create this rule. + * + * @return new rule instance to be used in unit tests + */ + public static EmbeddedElasticsearchPolicy create() { + return new EmbeddedElasticsearchPolicy(EmbeddedElasticsearchNode.create()); + } + + /** + * Exposes current ES transport client. + * + * @return initialized instance of ES + */ + Client client() { + return node.client(); + } + + /** + * HTTP address for rest clients (can be ES native or any other). + * + * @return HTTP hostname/port to connect to this ES instance + */ + TransportAddress httpAddress() { + return node.httpAddress(); + } +} + +// End EmbeddedElasticsearchPolicy.java http://git-wip-us.apache.org/repos/asf/calcite/blob/6e8bb5a1/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java ---------------------------------------------------------------------- diff --git a/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java deleted file mode 100644 index 86e9d9f..0000000 --- a/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.test; - -import com.google.common.base.Function; - -import java.util.List; - -import javax.annotation.Nullable; - -/** - * Internal util methods for ElasticSearch tests - */ -public class ElasticChecker { - - private ElasticChecker() {} - - - /** Returns a function that checks that a particular Elasticsearch pipeline is - * generated to implement a query. - * - * @param strings list of expected queries - * @return function to perform the check - */ - public static Function<List, Void> elasticsearchChecker(final String... strings) { - return new Function<List, Void>() { - @Nullable - @Override public Void apply(@Nullable List actual) { - Object[] actualArray = actual == null || actual.isEmpty() ? null - : ((List) actual.get(0)).toArray(); - CalciteAssert.assertArrayEqual("expected Elasticsearch query not found", strings, - actualArray); - return null; - } - }; - } -} - -// End ElasticChecker.java http://git-wip-us.apache.org/repos/asf/calcite/blob/6e8bb5a1/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticsearchChecker.java ---------------------------------------------------------------------- diff --git a/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticsearchChecker.java b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticsearchChecker.java new file mode 100644 index 0000000..d897dfb --- /dev/null +++ b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticsearchChecker.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test; + +import com.google.common.base.Function; + +import java.util.List; + +import javax.annotation.Nullable; + +/** + * Utility methods for Elasticsearch tests. + */ +public class ElasticsearchChecker { + + private ElasticsearchChecker() {} + + + /** Returns a function that checks that a particular Elasticsearch pipeline is + * generated to implement a query. + * + * @param strings list of expected queries + * @return function to perform the check + */ + public static Function<List, Void> elasticsearchChecker(final String... strings) { + return new Function<List, Void>() { + @Nullable + @Override public Void apply(@Nullable List actual) { + Object[] actualArray = actual == null || actual.isEmpty() ? null + : ((List) actual.get(0)).toArray(); + CalciteAssert.assertArrayEqual("expected Elasticsearch query not found", strings, + actualArray); + return null; + } + }; + } +} + +// End ElasticsearchChecker.java http://git-wip-us.apache.org/repos/asf/calcite/blob/6e8bb5a1/elasticsearch2/src/test/resources/zips-mini.json ---------------------------------------------------------------------- diff --git a/elasticsearch2/src/test/resources/zips-mini.json b/elasticsearch2/src/test/resources/zips-mini.json index d70eadc..858117a 100644 --- a/elasticsearch2/src/test/resources/zips-mini.json +++ b/elasticsearch2/src/test/resources/zips-mini.json @@ -146,4 +146,4 @@ { "_id" : "98310", "city" : "BREMERTON", "loc" : [ -122.629913, 47.601916 ], "pop" : 49057, "state" : "WA" } { "_id" : "99504", "city" : "ANCHORAGE", "loc" : [ -149.74467, 61.203696 ], "pop" : 32383, "state" : "AK" } { "_id" : "99709", "city" : "FAIRBANKS", "loc" : [ -147.846917, 64.85437 ], "pop" : 23238, "state" : "AK" } -{ "_id" : "99801", "city" : "JUNEAU", "loc" : [ -134.529429, 58.362767 ], "pop" : 24947, "state" : "AK" } \ No newline at end of file +{ "_id" : "99801", "city" : "JUNEAU", "loc" : [ -134.529429, 58.362767 ], "pop" : 24947, "state" : "AK" } http://git-wip-us.apache.org/repos/asf/calcite/blob/6e8bb5a1/elasticsearch5/src/test/java/org/apache/calcite/adapter/elasticsearch5/ElasticSearch5AdapterTest.java ---------------------------------------------------------------------- diff --git a/elasticsearch5/src/test/java/org/apache/calcite/adapter/elasticsearch5/ElasticSearch5AdapterTest.java b/elasticsearch5/src/test/java/org/apache/calcite/adapter/elasticsearch5/ElasticSearch5AdapterTest.java deleted file mode 100644 index 438e048..0000000 --- a/elasticsearch5/src/test/java/org/apache/calcite/adapter/elasticsearch5/ElasticSearch5AdapterTest.java +++ /dev/null @@ -1,401 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.adapter.elasticsearch5; - -import org.apache.calcite.jdbc.CalciteConnection; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.schema.impl.ViewTable; -import org.apache.calcite.schema.impl.ViewTableMacro; -import org.apache.calcite.test.CalciteAssert; -import org.apache.calcite.test.ElasticChecker; - -import com.google.common.io.LineProcessor; -import com.google.common.io.Resources; - -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentType; - -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.Collections; -import java.util.Locale; - -/** - * Set of tests for ES adapter. Uses real instance via {@link EmbeddedElasticRule}. Document - * source is local {@code zips-mini.json} file (located in the classpath). - */ -public class ElasticSearch5AdapterTest { - - @ClassRule //init once for all tests - public static final EmbeddedElasticRule NODE = EmbeddedElasticRule.create(); - - private static final String ZIPS = "zips"; - - /** - * Used to create {@code zips} index and insert some data - * - * @throws Exception when ES instance setup failed - */ - @BeforeClass - public static void setupInstance() throws Exception { - // define mapping so fields are searchable (term query) - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() - .startObject("properties") - .startObject("city").field("type", "string") - .field("index", "not_analyzed").endObject() - .startObject("state").field("type", "string") - .field("index", "not_analyzed").endObject() - .startObject("pop").field("type", "long").endObject() - .endObject() - .endObject(); - - // create index - NODE.client().admin().indices() - .prepareCreate(ZIPS) - .addMapping(ZIPS, mapping) - .get(); - - BulkRequestBuilder bulk = NODE.client().prepareBulk() - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - - // load records from file - Resources.readLines(ElasticSearch5AdapterTest.class.getResource("/zips-mini.json"), - StandardCharsets.UTF_8, new LineProcessor<Void>() { - @Override public boolean processLine(String line) throws IOException { - line = line.replaceAll("_id", "id"); // _id is a reserved attribute in ES - bulk.add(NODE.client().prepareIndex(ZIPS, ZIPS) - .setSource(line.getBytes(StandardCharsets.UTF_8), XContentType.JSON)); - return true; - } - - @Override public Void getResult() { - return null; - } - }); - - if (bulk.numberOfActions() == 0) { - throw new IllegalStateException("No records to be indexed"); - } - - BulkResponse response = bulk.execute().get(); - - if (response.hasFailures()) { - throw new IllegalStateException( - String.format(Locale.getDefault(), "Failed to populate %s:\n%s", NODE.httpAddress(), - Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed) - .map(BulkItemResponse::getFailureMessage).findFirst().orElse("<unknown>"))); - } - - } - - private CalciteAssert.ConnectionFactory newConnectionFactory() { - return new CalciteAssert.ConnectionFactory() { - @Override public Connection createConnection() throws SQLException { - final Connection connection = DriverManager.getConnection("jdbc:calcite:"); - final SchemaPlus root = connection.unwrap(CalciteConnection.class).getRootSchema(); - - root.add("elastic", new Elasticsearch5Schema(NODE.client(), ZIPS)); - - // add calcite view programmatically - final String viewSql = "select cast(_MAP['city'] AS varchar(20)) AS \"city\", " - + " cast(_MAP['loc'][0] AS float) AS \"longitude\",\n" - + " cast(_MAP['loc'][1] AS float) AS \"latitude\",\n" - + " cast(_MAP['pop'] AS integer) AS \"pop\", " - + " cast(_MAP['state'] AS varchar(2)) AS \"state\", " - + " cast(_MAP['id'] AS varchar(5)) AS \"id\" " - + "from \"elastic\".\"zips\""; - - ViewTableMacro macro = ViewTable.viewMacro(root, viewSql, - Collections.singletonList("elastic"), Arrays.asList("elastic", "view"), false); - root.add("ZIPS", macro); - - return connection; - } - }; - } - - private CalciteAssert.AssertThat calciteAssert() { - return CalciteAssert.that() - .with(newConnectionFactory()); - } - - /** - * Tests using calcite view - */ - @Test - public void view() { - calciteAssert() - .query("select * from zips where \"city\" = 'BROOKLYN'") - .returns("city=BROOKLYN; longitude=-73.956985; latitude=40.646694; " - + "pop=111396; state=NY; id=11226\n") - .returnsCount(1); - } - - @Test - public void emptyResult() { - CalciteAssert.that() - .with(newConnectionFactory()) - .query("select * from zips limit 0") - .returnsCount(0); - - CalciteAssert.that() - .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" where _MAP['Foo'] = '_MISSING_'") - .returnsCount(0); - } - - @Test - public void basic() throws Exception { - CalciteAssert.that() - .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" where _MAP['city'] = 'BROOKLYN'") - .returnsCount(1); - - CalciteAssert.that() - .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" where" - + " _MAP['city'] in ('BROOKLYN', 'WASHINGTON')") - .returnsCount(2); - - // lower-case - CalciteAssert.that() - .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" where " - + "_MAP['city'] in ('brooklyn', 'Brooklyn', 'BROOK') ") - .returnsCount(0); - - // missing field - CalciteAssert.that() - .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" where _MAP['CITY'] = 'BROOKLYN'") - .returnsCount(0); - - // limit works - CalciteAssert.that() - .with(newConnectionFactory()) - .query("select * from \"elastic\".\"zips\" limit 42") - .returnsCount(42); - - } - - @Test public void testSort() { - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n" - + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n" - + " ElasticsearchProject(city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'), 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT], pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n" - + " ElasticsearchTableScan(table=[[elastic, zips]])"; - - calciteAssert() - .query("select * from zips order by \"state\"") - .returnsCount(10) - .explainContains(explain); - } - - @Test public void testSortLimit() { - final String sql = "select \"state\", \"pop\" from zips\n" - + "order by \"state\", \"pop\" offset 2 rows fetch next 3 rows only"; - calciteAssert() - .query(sql) - .returnsUnordered("state=AK; pop=32383", - "state=AL; pop=42124", - "state=AL; pop=43862") - .queryContains( - ElasticChecker.elasticsearchChecker( - "\"_source\" : [\"state\", \"pop\"]", - "\"sort\": [ {\"state\": \"asc\"}, {\"pop\": \"asc\"}]", - "\"from\": 2", - "\"size\": 3")); - } - - - - @Test public void testOffsetLimit() { - final String sql = "select \"state\", \"id\" from zips\n" - + "offset 2 fetch next 3 rows only"; - calciteAssert() - .query(sql) - .runs() - .queryContains( - ElasticChecker.elasticsearchChecker( - "\"from\": 2", - "\"size\": 3", - "\"_source\" : [\"state\", \"id\"]")); - } - - @Test public void testLimit() { - final String sql = "select \"state\", \"id\" from zips\n" - + "fetch next 3 rows only"; - - calciteAssert() - .query(sql) - .runs() - .queryContains( - ElasticChecker.elasticsearchChecker( - "\"size\": 3", - "\"_source\" : [\"state\", \"id\"]")); - } - - @Test public void testFilterSort() { - final String sql = "select * from zips\n" - + "where \"state\" = 'CA' and \"pop\" >= 94000\n" - + "order by \"state\", \"pop\""; - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n" - + " ElasticsearchSort(sort0=[$4], sort1=[$3], dir0=[ASC], dir1=[ASC])\n" - + " ElasticsearchProject(city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'), 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT], pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n" - + " ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'CA'), >=(CAST(ITEM($0, 'pop')):INTEGER, 94000))])\n" - + " ElasticsearchTableScan(table=[[elastic, zips]])\n\n"; - calciteAssert() - .query(sql) - .returnsOrdered("city=NORWALK; longitude=-118.081767; latitude=33.90564;" - + " pop=94188; state=CA; id=90650", - "city=LOS ANGELES; longitude=-118.258189; latitude=34.007856;" - + " pop=96074; state=CA; id=90011", - "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177;" - + " pop=99568; state=CA; id=90201") - .queryContains( - ElasticChecker.elasticsearchChecker("\"query\" : " - + "{\"constant_score\":{\"filter\":{\"bool\":" - + "{\"must\":[{\"term\":{\"state\":\"CA\"}}," - + "{\"range\":{\"pop\":{\"gte\":94000}}}]}}}}", - "\"script_fields\": {\"longitude\":{\"script\":\"params._source.loc[0]\"}, " - + "\"latitude\":{\"script\":\"params._source.loc[1]\"}, " - + "\"city\":{\"script\": \"params._source.city\"}, " - + "\"pop\":{\"script\": \"params._source.pop\"}, " - + "\"state\":{\"script\": \"params._source.state\"}, " - + "\"id\":{\"script\": \"params._source.id\"}}", - "\"sort\": [ {\"state\": \"asc\"}, {\"pop\": \"asc\"}]")) - .explainContains(explain); - } - - @Test public void testFilterSortDesc() { - final String sql = "select * from zips\n" - + "where \"pop\" BETWEEN 95000 AND 100000\n" - + "order by \"state\" desc, \"pop\""; - calciteAssert() - .query(sql) - .limit(4) - .returnsOrdered( - "city=LOS ANGELES; longitude=-118.258189; latitude=34.007856; pop=96074; state=CA; id=90011", - "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177; pop=99568; state=CA; id=90201"); - } - - @Test public void testFilterRedundant() { - final String sql = "select * from zips\n" - + "where \"state\" > 'CA' and \"state\" < 'AZ' and \"state\" = 'OK'"; - calciteAssert() - .query(sql) - .runs() - .queryContains( - ElasticChecker.elasticsearchChecker("" - + "\"query\" : {\"constant_score\":{\"filter\":{\"bool\":" - + "{\"must\":[{\"term\":{\"state\":\"OK\"}}]}}}}", - "\"script_fields\": {\"longitude\":{\"script\":\"params._source.loc[0]\"}, " - + "\"latitude\":{\"script\":\"params._source.loc[1]\"}, " - + "\"city\":{\"script\": \"params._source.city\"}, " - + "\"pop\":{\"script\": \"params._source.pop\"}, \"state\":{\"script\": \"params._source.state\"}, " - + "\"id\":{\"script\": \"params._source.id\"}}" - )); - } - - @Test public void testInPlan() { - final String[] searches = { - "\"query\" : {\"constant_score\":{\"filter\":{\"bool\":{\"should\":" - + "[{\"bool\":{\"must\":[{\"term\":{\"pop\":96074}}]}},{\"bool\":{\"must\":[{\"term\":" - + "{\"pop\":99568}}]}}]}}}}", - "\"script_fields\": {\"longitude\":{\"script\":\"params._source.loc[0]\"}, " - + "\"latitude\":{\"script\":\"params._source.loc[1]\"}, " - + "\"city\":{\"script\": \"params._source.city\"}, " - + "\"pop\":{\"script\": \"params._source.pop\"}, " - + "\"state\":{\"script\": \"params._source.state\"}, " - + "\"id\":{\"script\": \"params._source.id\"}}" - }; - - calciteAssert() - .query("select * from zips where \"pop\" in (96074, 99568)") - .returnsUnordered( - "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177; pop=99568; state=CA; id=90201", - "city=LOS ANGELES; longitude=-118.258189; latitude=34.007856; pop=96074; state=CA; id=90011" - ) - .queryContains(ElasticChecker.elasticsearchChecker(searches)); - } - - @Test public void testZips() { - calciteAssert() - .query("select \"state\", \"city\" from zips") - .returnsCount(10); - } - - @Test public void testProject() { - final String sql = "select \"state\", \"city\", 0 as \"zero\"\n" - + "from zips\n" - + "order by \"state\", \"city\""; - - calciteAssert() - .query(sql) - .limit(2) - .returnsUnordered("state=AK; city=ANCHORAGE; zero=0", - "state=AK; city=FAIRBANKS; zero=0") - .queryContains( - ElasticChecker.elasticsearchChecker("\"script_fields\": " - + "{\"zero\":{\"script\": \"0\"}, " - + "\"state\":{\"script\": \"params._source.state\"}, " - + "\"city\":{\"script\": \"params._source.city\"}}", - "\"sort\": [ {\"state\": \"asc\"}, {\"city\": \"asc\"}]")); - } - - @Test public void testFilter() { - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n" - + " ElasticsearchProject(state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n" - + " ElasticsearchFilter(condition=[=(CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'CA')])\n" - + " ElasticsearchTableScan(table=[[elastic, zips]])"; - calciteAssert() - .query("select \"state\", \"city\" from zips where \"state\" = 'CA'") - .limit(3) - .returnsUnordered("state=CA; city=BELL GARDENS", - "state=CA; city=LOS ANGELES", - "state=CA; city=NORWALK") - .explainContains(explain); - } - - @Test public void testFilterReversed() { - calciteAssert() - .query("select \"state\", \"city\" from zips where 'WI' < \"state\" order by \"city\"") - .limit(2) - .returnsUnordered("state=WV; city=BECKLEY", - "state=WY; city=CHEYENNE"); - calciteAssert() - .query("select \"state\", \"city\" from zips where \"state\" > 'WI' order by \"city\"") - .limit(2) - .returnsUnordered("state=WV; city=BECKLEY", - "state=WY; city=CHEYENNE"); - } - -} - -// End ElasticSearch5AdapterTest.java
