This is an automated email from the ASF dual-hosted git repository. zabetak pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/calcite.git
commit ab19f98172848fe303a18173946c2def0b0d0312 Author: Stamatis Zampetakis <[email protected]> AuthorDate: Wed Sep 23 20:13:07 2020 +0200 [CALCITE-4278] Add Druid adapter tests in GitHub CI Skip failing tests in Druid due to CALCITE-4279 Skip part of DruidAdapterIT#testAggSameColumnMultipleTimes due to CALCITE-4204 Close apache/calcite#2169 --- .github/workflows/main.yml | 40 ++++++++++++++++++++++ .../src/main/java/org/apache/calcite/util/Bug.java | 5 +++ .../org/apache/calcite/test/DruidAdapter2IT.java | 20 +++++++++++ .../org/apache/calcite/test/DruidAdapterIT.java | 28 +++++++++++++-- 4 files changed, 90 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5c04b01..68558d8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -129,3 +129,43 @@ jobs: with: job-id: jdk8 arguments: --scan --no-parallel --no-daemon testSlow + linux-druid: + if: github.event.action != 'labeled' + name: 'Linux (JDK 8) Druid Tests' + runs-on: ubuntu-latest + steps: + - name: 'Set up JDK 8' + uses: actions/setup-java@v1 + with: + java-version: 8 + - name: 'Checkout Druid dataset' + uses: actions/checkout@master + with: + repository: zabetak/calcite-druid-dataset + fetch-depth: 1 + path: druid-dataset + - name: 'Start Druid containers' + working-directory: ./druid-dataset + run: | + chmod -R 777 storage + docker-compose up -d + - name: 'Wait Druid nodes to startup' + run: | + until docker logs coordinator | grep "Successfully started lifecycle \[module\]"; do sleep 1s; done + until docker logs router | grep "Successfully started lifecycle \[module\]"; do sleep 1s; done + until docker logs historical | grep "Successfully started lifecycle \[module\]"; do sleep 1s; done + until docker logs middlemanager | grep "Successfully started lifecycle \[module\]"; do sleep 1s; done + until docker logs broker | grep "Successfully started lifecycle \[module\]"; do sleep 1s; done + - name: 'Index Foodmart/Wikipedia datasets' + working-directory: ./druid-dataset + run: ./index.sh 30s + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + path: calcite + - uses: burrunan/gradle-cache-action@v1 + name: 'Run Druid tests' + with: + build-root-directory: ./calcite + job-id: Druid8 + arguments: --scan --no-parallel --no-daemon :druid:test -Dcalcite.test.druid=true diff --git a/core/src/main/java/org/apache/calcite/util/Bug.java b/core/src/main/java/org/apache/calcite/util/Bug.java index 85531a2..25ca437 100644 --- a/core/src/main/java/org/apache/calcite/util/Bug.java +++ b/core/src/main/java/org/apache/calcite/util/Bug.java @@ -201,6 +201,11 @@ public abstract class Bug { * fixed. */ public static final boolean CALCITE_4213_FIXED = false; + /** Whether + * <a href="https://issues.apache.org/jira/browse/CALCITE-4279";>[CALCITE-4279] + * SEARCH operator cannot be pushed into Druid</a> is fixed. */ + public static final boolean CALCITE_4279_FIXED = false; + /** * Use this to flag temporary code. */ diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java index b48519e..913bd82 100644 --- a/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java +++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java @@ -501,6 +501,7 @@ public class DruidAdapter2IT { /** Tests a query that contains no GROUP BY and is therefore executed as a * Druid "select" query. */ @Test void testFilterSortDesc() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); final String sql = "select \"product_name\" from \"foodmart\"\n" + "where \"product_id\" BETWEEN '1500' AND '1502'\n" + "order by \"state_province\" desc, \"product_id\""; @@ -856,6 +857,7 @@ public class DruidAdapter2IT { } @Test void testGroupByMonthGranularityFiltered() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); final String sql = "select sum(\"unit_sales\") as s,\n" + " count(\"store_sqft\") as c\n" + "from \"foodmart\"\n" @@ -906,6 +908,7 @@ public class DruidAdapter2IT { } @Test void testTopNDayGranularityFiltered() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); final String sql = "select sum(\"unit_sales\") as s,\n" + "max(\"unit_sales\") as m,\n" + "\"state_province\" as p\n" @@ -1009,6 +1012,7 @@ public class DruidAdapter2IT { } @Test void testFilterDistinct() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); final String sql = "select distinct \"state_province\", \"city\",\n" + " \"product_name\"\n" + "from \"foodmart\"\n" @@ -1047,6 +1051,7 @@ public class DruidAdapter2IT { } @Test void testFilter() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); final String sql = "select \"state_province\", \"city\",\n" + " \"product_name\"\n" + "from \"foodmart\"\n" @@ -1159,6 +1164,7 @@ public class DruidAdapter2IT { } @Test void testPushAggregateOnTimeWithExtractYear() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select EXTRACT( year from \"timestamp\") as \"year\",\"product_id\" from " + "\"foodmart\" where \"product_id\" = 1016 and " + "\"timestamp\" < cast('1999-01-02' as timestamp) and \"timestamp\" > cast" @@ -1176,6 +1182,7 @@ public class DruidAdapter2IT { } @Test void testPushAggregateOnTimeWithExtractMonth() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select EXTRACT( month from \"timestamp\") as \"month\",\"product_id\" from " + "\"foodmart\" where \"product_id\" = 1016 and " + "\"timestamp\" < cast('1997-06-02' as timestamp) and \"timestamp\" > cast" @@ -1194,6 +1201,7 @@ public class DruidAdapter2IT { } @Test void testPushAggregateOnTimeWithExtractDay() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select EXTRACT( day from \"timestamp\") as \"day\"," + "\"product_id\" from \"foodmart\"" + " where \"product_id\" = 1016 and " @@ -1213,6 +1221,7 @@ public class DruidAdapter2IT { } @Test void testPushAggregateOnTimeWithExtractHourOfDay() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select EXTRACT( hour from \"timestamp\") as \"hourOfDay\",\"product_id\" from " + "\"foodmart\" where \"product_id\" = 1016 and " @@ -1225,6 +1234,7 @@ public class DruidAdapter2IT { } @Test void testPushAggregateOnTimeWithExtractYearMonthDay() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select EXTRACT( day from \"timestamp\") as \"day\", EXTRACT( month from " + "\"timestamp\") as \"month\", EXTRACT( year from \"timestamp\") as \"year\",\"" + "product_id\" from \"foodmart\" where \"product_id\" = 1016 and " @@ -1259,6 +1269,7 @@ public class DruidAdapter2IT { } @Test void testPushAggregateOnTimeWithExtractYearMonthDayWithOutRenaming() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select EXTRACT( day from \"timestamp\"), EXTRACT( month from " + "\"timestamp\"), EXTRACT( year from \"timestamp\"),\"" + "product_id\" from \"foodmart\" where \"product_id\" = 1016 and " @@ -1292,6 +1303,7 @@ public class DruidAdapter2IT { } @Test void testPushAggregateOnTimeWithExtractWithOutRenaming() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select EXTRACT( day from \"timestamp\"), " + "\"product_id\" as \"dayOfMonth\" from \"foodmart\" " + "where \"product_id\" = 1016 and \"timestamp\" < cast('1997-01-20' as timestamp) " @@ -1316,6 +1328,7 @@ public class DruidAdapter2IT { } @Test void testPushComplexFilter() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select sum(\"store_sales\") from \"foodmart\" " + "where EXTRACT( year from \"timestamp\") = 1997 and " + "\"cases_per_pallet\" >= 8 and \"cases_per_pallet\" <= 10 and " @@ -2035,6 +2048,7 @@ public class DruidAdapter2IT { } @Test void testOrderByOnMetricsInSelectDruidQuery() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); final String sqlQuery = "select \"store_sales\" as a, \"store_cost\" as b, \"store_sales\" - " + "\"store_cost\" as c from \"foodmart\" where \"timestamp\" " + ">= '1997-01-01 00:00:00' and \"timestamp\" < '1997-09-01 00:00:00' order by c " @@ -2237,6 +2251,7 @@ public class DruidAdapter2IT { /** Tests that multiple aggregates with filter clauses have their filters extracted to the outer filter field for data pruning. */ @Test void testFilterClausesFactoredForPruning1() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select " + "sum(\"store_sales\") filter (where \"store_state\" = 'CA'), " + "sum(\"store_sales\") filter (where \"store_state\" = 'WA') " @@ -2261,6 +2276,7 @@ public class DruidAdapter2IT { * extracted to the outer filter field for data pruning in the presence of an * outer filter. */ @Test void testFilterClausesFactoredForPruning2() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select " + "sum(\"store_sales\") filter (where \"store_state\" = 'CA'), " + "sum(\"store_sales\") filter (where \"store_state\" = 'WA') " @@ -2411,6 +2427,7 @@ public class DruidAdapter2IT { /** Tests that an aggregate with a nested filter clause has its filter * factored out. */ @Test void testNestedFilterClauseFactored() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); // Logically equivalent to // select sum("store_sales") from "foodmart" where "store_state" in ('CA', 'OR') String sql = @@ -2701,6 +2718,7 @@ public class DruidAdapter2IT { } @Test void testFilterWithFloorOnTime() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); // Test filter on floor on time column is pushed to druid final String sql = "Select floor(\"timestamp\" to MONTH) as t from \"foodmart\" where " @@ -2755,6 +2773,7 @@ public class DruidAdapter2IT { } @Test void testFloorToDateRangeWithTimeZone() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); final String sql = "Select floor(\"timestamp\" to MONTH) as t from " + "\"foodmart\" where floor(\"timestamp\" to MONTH) >= '1997-05-01 00:00:00' " + "and floor(\"timestamp\" to MONTH) < '1997-05-02 00:00:00' order by t" @@ -3173,6 +3192,7 @@ public class DruidAdapter2IT { @Test void testCeilFilterExpression() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); final String sql = "SELECT COUNT(*) FROM " + FOODMART_TABLE + " WHERE ceil(\"store_sales\") > 1" + " AND ceil(\"timestamp\" TO DAY) < CAST('1997-01-05' AS TIMESTAMP)" + " AND ceil(\"timestamp\" TO MONTH) < CAST('1997-03-01' AS TIMESTAMP)" diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java index c16b77d..f1a0e68 100644 --- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java +++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java @@ -764,6 +764,7 @@ public class DruidAdapterIT { /** Tests a query that contains no GROUP BY and is therefore executed as a * Druid "select" query. */ @Test void testFilterSortDesc() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); final String sql = "select \"product_name\" from \"foodmart\"\n" + "where \"product_id\" BETWEEN '1500' AND '1502'\n" + "order by \"state_province\" desc, \"product_id\""; @@ -1121,6 +1122,7 @@ public class DruidAdapterIT { } @Test void testGroupByMonthGranularityFiltered() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); final String sql = "select sum(\"unit_sales\") as s,\n" + " count(\"store_sqft\") as c\n" + "from \"foodmart\"\n" @@ -1171,6 +1173,7 @@ public class DruidAdapterIT { } @Test void testTopNDayGranularityFiltered() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); final String sql = "select sum(\"unit_sales\") as s,\n" + "max(\"unit_sales\") as m,\n" + "\"state_province\" as p\n" @@ -1276,6 +1279,7 @@ public class DruidAdapterIT { } @Test void testFilterDistinct() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); final String sql = "select distinct \"state_province\", \"city\",\n" + " \"product_name\"\n" + "from \"foodmart\"\n" @@ -1314,6 +1318,7 @@ public class DruidAdapterIT { } @Test void testFilter() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); final String sql = "select \"state_province\", \"city\",\n" + " \"product_name\"\n" + "from \"foodmart\"\n" @@ -1394,6 +1399,7 @@ public class DruidAdapterIT { /** Tests a query that exposed several bugs in the interpreter. */ @Test void testWhereGroupBy() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select \"wikipedia\".\"countryName\" as \"c0\",\n" + " sum(\"wikipedia\".\"count\") as \"m1\",\n" + " sum(\"wikipedia\".\"deleted\") as \"m2\",\n" @@ -1443,6 +1449,7 @@ public class DruidAdapterIT { } @Test void testPushAggregateOnTimeWithExtractYear() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select EXTRACT( year from \"timestamp\") as \"year\",\"product_id\" from " + "\"foodmart\" where \"product_id\" = 1016 and " + "\"timestamp\" < cast('1999-01-02' as timestamp) and \"timestamp\" > cast" @@ -1460,6 +1467,7 @@ public class DruidAdapterIT { } @Test void testPushAggregateOnTimeWithExtractMonth() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select EXTRACT( month from \"timestamp\") as \"month\",\"product_id\" from " + "\"foodmart\" where \"product_id\" = 1016 and " + "\"timestamp\" < cast('1997-06-02' as timestamp) and \"timestamp\" > cast" @@ -1478,6 +1486,7 @@ public class DruidAdapterIT { } @Test void testPushAggregateOnTimeWithExtractDay() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select EXTRACT( day from \"timestamp\") as \"day\"," + "\"product_id\" from \"foodmart\"" + " where \"product_id\" = 1016 and " @@ -1497,6 +1506,7 @@ public class DruidAdapterIT { } @Test void testPushAggregateOnTimeWithExtractHourOfDay() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select EXTRACT( hour from \"timestamp\") as \"hourOfDay\",\"product_id\" from " + "\"foodmart\" where \"product_id\" = 1016 and " @@ -1509,6 +1519,7 @@ public class DruidAdapterIT { } @Test void testPushAggregateOnTimeWithExtractYearMonthDay() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select EXTRACT( day from \"timestamp\") as \"day\", EXTRACT( month from " + "\"timestamp\") as \"month\", EXTRACT( year from \"timestamp\") as \"year\",\"" + "product_id\" from \"foodmart\" where \"product_id\" = 1016 and " @@ -1543,6 +1554,7 @@ public class DruidAdapterIT { } @Test void testPushAggregateOnTimeWithExtractYearMonthDayWithOutRenaming() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select EXTRACT( day from \"timestamp\"), EXTRACT( month from " + "\"timestamp\"), EXTRACT( year from \"timestamp\"),\"" + "product_id\" from \"foodmart\" where \"product_id\" = 1016 and " @@ -1576,6 +1588,7 @@ public class DruidAdapterIT { } @Test void testPushAggregateOnTimeWithExtractWithOutRenaming() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select EXTRACT( day from \"timestamp\"), " + "\"product_id\" as \"dayOfMonth\" from \"foodmart\" " + "where \"product_id\" = 1016 and \"timestamp\" < cast('1997-01-20' as timestamp) " @@ -1600,6 +1613,7 @@ public class DruidAdapterIT { } @Test void testPushComplexFilter() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select sum(\"store_sales\") from \"foodmart\" " + "where EXTRACT( year from \"timestamp\") = 1997 and " + "\"cases_per_pallet\" >= 8 and \"cases_per_pallet\" <= 10 and " @@ -2335,6 +2349,7 @@ public class DruidAdapterIT { } @Test void testOrderByOnMetricsInSelectDruidQuery() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); final String sqlQuery = "select \"store_sales\" as a, \"store_cost\" as b, \"store_sales\" - " + "\"store_cost\" as c from \"foodmart\" where \"timestamp\" " + ">= '1997-01-01 00:00:00 UTC' and \"timestamp\" < '1997-09-01 00:00:00 UTC' order by c " @@ -2537,6 +2552,7 @@ public class DruidAdapterIT { /** Tests that multiple aggregates with filter clauses have their filters * extracted to the outer filter field for data pruning. */ @Test void testFilterClausesFactoredForPruning1() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select " + "sum(\"store_sales\") filter (where \"store_state\" = 'CA'), " + "sum(\"store_sales\") filter (where \"store_state\" = 'WA') " @@ -2561,6 +2577,7 @@ public class DruidAdapterIT { * extracted to the outer filter field for data pruning in the presence of an * outer filter. */ @Test void testFilterClausesFactoredForPruning2() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); String sql = "select " + "sum(\"store_sales\") filter (where \"store_state\" = 'CA'), " + "sum(\"store_sales\") filter (where \"store_state\" = 'WA') " @@ -2710,6 +2727,7 @@ public class DruidAdapterIT { /** Tests that an aggregate with a nested filter clause has its filter * factored out. */ @Test void testNestedFilterClauseFactored() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); // Logically equivalent to // select sum("store_sales") from "foodmart" where "store_state" in ('CA', 'OR') String sql = @@ -3167,6 +3185,7 @@ public class DruidAdapterIT { } @Test void testFilterWithFloorOnTime() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); // Test filter on floor on time column is pushed to druid final String sql = "Select cast(floor(\"timestamp\" to MONTH) as timestamp) as t from \"foodmart\" where " @@ -3294,6 +3313,7 @@ public class DruidAdapterIT { } @Test void testFloorToDateRangeWithTimeZone() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); final String sql = "Select cast(floor(\"timestamp\" to MONTH) as timestamp) as t from " + "\"foodmart\" where floor(\"timestamp\" to MONTH) >= '1997-05-01 00:00:00 Asia/Kolkata' " + "and floor(\"timestamp\" to MONTH) < '1997-05-02 00:00:00 Asia/Kolkata' order by t" @@ -3797,6 +3817,7 @@ public class DruidAdapterIT { @Test void testCeilFilterExpression() { + Assumptions.assumeTrue(Bug.CALCITE_4279_FIXED, "CALCITE-4279"); final String sql = "SELECT COUNT(*) FROM " + FOODMART_TABLE + " WHERE ceil(\"store_sales\") > 1" + " AND ceil(\"timestamp\" TO DAY) < CAST('1997-01-05' AS TIMESTAMP)" + " AND ceil(\"timestamp\" TO MONTH) < CAST('1997-03-01' AS TIMESTAMP)" @@ -4349,14 +4370,15 @@ public class DruidAdapterIT { "SELECT \"product_id\" as prod_id1, \"product_id\" as prod_id2, " + "SUM(\"store_sales\") as S1, SUM(\"store_sales\") as S2 FROM " + FOODMART_TABLE + " GROUP BY \"product_id\" ORDER BY prod_id2 LIMIT 1"; - sql(sql, FOODMART) + CalciteAssert.AssertQuery q = sql(sql, FOODMART) .explainContains("BindableProject(PROD_ID1=[$0], PROD_ID2=[$0], S1=[$1], S2=[$1])\n" + " DruidQuery(table=[[foodmart, foodmart]], intervals=[[1900-01-09T00:00:00.000Z/" + "2992-01-10T00:00:00.000Z]], projects=[[$1, $90]], groups=[{0}], aggs=[[SUM($1)]], " + "sort0=[0], dir0=[ASC], fetch=[1])") .queryContains( - new DruidChecker("\"queryType\":\"groupBy\"")) - .returnsOrdered("PROD_ID1=1; PROD_ID2=1; S1=236.55; S2=236.55"); + new DruidChecker("\"queryType\":\"groupBy\"")); + Assumptions.assumeTrue(Bug.CALCITE_4204_FIXED, "CALCITE-4204"); + q.returnsOrdered("PROD_ID1=1; PROD_ID2=1; S1=236.55; S2=236.55"); } @Test void testGroupBy1() {
