[ASTERIXDB-2170][SQL] Fix resolution order of implicit field access - user model changes: yes - storage format changes: no - interface changes: no
Details: - Improved name resolution rules - Resolve field access to the nearest variable in scope instead of raising compile-time error - Do not rely on type information when resolving names - Cleanup group variable handling in GroupBy clause, no longer use âwithâ map for it - Fix ByNameToByIndexFieldAccessRule to use type environment of its input operator when analyzing its expression - Fix ExternalGroupByPOperator to use input schema of its aggregate function when generating runtime for that function - Fix invalid free variable computation for GroupBy clause Change-Id: I50bc823ff53da06507a5454b30f4f500b862d4bf Reviewed-on: https://asterix-gerrit.ics.uci.edu/2207 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Dmitry Lychagin <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/ef1719e3 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/ef1719e3 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/ef1719e3 Branch: refs/heads/master Commit: ef1719e3eac51eb659eb0ebae3584d25dea85189 Parents: ed50308 Author: Dmitry Lychagin <[email protected]> Authored: Fri Dec 15 08:33:07 2017 -0800 Committer: Dmitry Lychagin <[email protected]> Committed: Fri Dec 15 17:28:40 2017 -0800 ---------------------------------------------------------------------- .../asterix/optimizer/base/RuleCollections.java | 2 - .../rules/ByNameToByIndexFieldAccessRule.java | 2 +- .../PushAggregateIntoNestedSubplanRule.java | 68 ++-- .../optimizer/rules/ResolveVariableRule.java | 335 ------------------- .../LangExpressionToPlanTranslator.java | 77 +++-- .../SqlppExpressionToPlanTranslator.java | 8 +- .../asterix/test/sqlpp/ParserTestExecutor.java | 8 +- .../queries_sqlpp/denorm-cust-order.sqlpp | 4 +- .../queries_sqlpp/orderby-desc-using-gby.sqlpp | 2 +- .../queries_sqlpp/orders-aggreg.sqlpp | 12 +- .../q01_pricing_summary_report_nt.sqlpp | 32 +- .../queries_sqlpp/query-issue562.sqlpp | 8 +- .../split-materialization-above-join.sqlpp | 4 +- .../results/aggregate/constant-gby-agg.plan | 6 +- .../results/gby_partitioning_property_01.plan | 10 +- .../results/hints/broadcast_hint_1.plan | 4 +- .../results/hints/broadcast_hint_2.plan | 4 +- .../results/hints/broadcast_hint_3.plan | 8 +- .../resources/optimizerts/results/loj-core.plan | 10 +- .../optimizerts/results/loj-sugar.plan | 10 +- .../btree-equi-join-non-enforced-05.plan | 10 +- .../btree-equi-join-non-enforced-06.plan | 10 +- .../btree-equi-join-non-enforced-07.plan | 10 +- .../btree-equi-join-non-enforced-08.plan | 6 +- .../btree-equi-join-non-enforced-09.plan | 6 +- .../btree-index-non-enforced-04.plan | 6 +- .../btree-index-non-enforced-05.plan | 6 +- .../btree-index-non-enforced-06.plan | 6 +- .../btree-index-non-enforced-07.plan | 6 +- .../btree-index-non-enforced-08.plan | 6 +- .../btree-index-non-enforced-09.plan | 6 +- .../btree-index-non-enforced-10.plan | 6 +- .../btree-index-non-enforced-105.plan | 8 +- .../btree-index-non-enforced-11.plan | 6 +- .../results/query-ASTERIXDB-159-3.plan | 8 +- .../results/query-ASTERIXDB-1671.plan | 4 +- .../results/query-ASTERIXDB-1806.plan | 6 +- .../results/query-ASTERIXDB-810-2.plan | 10 +- .../results/query-ASTERIXDB-810-3.plan | 10 +- .../results/query-ASTERIXDB-810.plan | 10 +- .../optimizerts/results/subquery/exists.plan | 20 +- .../optimizerts/results/subquery/in.plan | 10 +- .../results/subquery/in_correlated.plan | 10 +- .../results/subquery/not_exists.plan | 20 +- .../results/subquery/query-ASTERIXDB-1572.plan | 20 +- .../results/tpcds/query-ASTERIXDB-1580.plan | 28 +- .../tpcds/query-ASTERIXDB-1581-correlated.plan | 64 ++-- .../results/tpcds/query-ASTERIXDB-1581.plan | 26 +- .../results/tpcds/query-ASTERIXDB-1591.plan | 60 ++-- .../results/tpcds/query-ASTERIXDB-1596.plan | 10 +- .../optimizerts/results/tpch/q12_shipping.plan | 12 +- .../results/tpch/q12_shipping_broadcast.plan | 10 +- .../btree-index-join/secondary-equi-join_06.ast | 12 +- .../btree-index/btree-secondary-64.ast | 5 + .../btree-index/btree-secondary-65.ast | 5 + .../btree-index/btree-secondary-66.ast | 5 + .../btree-index/btree-secondary-67.ast | 5 + .../results_parser_sqlpp/count-tweets.ast | 1 + .../results_parser_sqlpp/denorm-cust-order.ast | 42 ++- .../results_parser_sqlpp/distinct_aggregate.ast | 6 +- .../results_parser_sqlpp/fj-dblp-csx.ast | 6 + .../inlined_q18_large_volume_customer.ast | 4 +- .../orderby-desc-using-gby.ast | 18 +- .../results_parser_sqlpp/orders-aggreg.ast | 26 +- .../q01_pricing_summary_report_nt.ast | 87 ++--- .../optimizerts/results_parser_sqlpp/q2.ast | 17 +- .../results_parser_sqlpp/query-issue562.ast | 17 +- .../rtree-index-join/query-issue838.ast | 5 +- .../split-materialization-above-join.ast | 56 +--- .../parserts/queries_sqlpp/columnalias2.sqlpp | 6 +- .../parserts/results_parser_sqlpp/2.ast | 33 +- .../parserts/results_parser_sqlpp/LetFor.ast | 2 +- .../results_parser_sqlpp/columnalias.ast | 24 +- .../results_parser_sqlpp/columnalias2.ast | 92 +++-- .../results_parser_sqlpp/columnalias3.ast | 24 +- .../results_parser_sqlpp/functionDecl1.ast | 2 +- .../results_parser_sqlpp/nestedFLWOGR.ast | 2 +- .../results_parser_sqlpp/nestedFLWOGR3.ast | 33 +- .../custord/join_q_09/join_q_09.2.query.sqlpp | 4 +- .../queries_sqlpp/dapd/q2-4/q2-4.3.query.sqlpp | 4 +- .../dblp-3_1.1/dblp-3_1.1.3.query.sqlpp | 2 +- .../dblp-3_1.2/dblp-3_1.2.3.query.sqlpp | 2 +- .../fuzzyjoin/dblp-3_1/dblp-3_1.3.query.sqlpp | 2 +- .../dblp-csx-2_5.1/dblp-csx-2_5.1.3.query.sqlpp | 2 +- .../dblp-csx-2_5.2/dblp-csx-2_5.2.3.query.sqlpp | 2 +- .../dblp-csx-2_5.3/dblp-csx-2_5.3.3.query.sqlpp | 2 +- .../dblp-csx-3_5.1/dblp-csx-3_5.1.3.query.sqlpp | 2 +- .../dblp-csx-3_5.2/dblp-csx-3_5.2.3.query.sqlpp | 2 +- .../dblp-csx-3_5.3.1.3.query.sqlpp | 2 +- .../dblp-csx-3_5.4.1.3.query.sqlpp | 2 +- .../dblp-csx-3_5.4/dblp-csx-3_5.4.3.query.sqlpp | 2 +- .../all_datasets/all_datasets.13.query.sqlpp | 14 +- .../all_datasets/all_datasets.3.query.sqlpp | 14 +- .../all_datasets/all_datasets.7.query.sqlpp | 14 +- .../single_dataverse.13.query.sqlpp | 14 +- .../single_dataverse.3.query.sqlpp | 14 +- .../single_dataverse.7.query.sqlpp | 14 +- .../conflict-fields-dataset.3.query.sqlpp | 3 +- .../groupby_rename_with_sugar.1.ddl.sqlpp | 28 ++ .../groupby_rename_with_sugar.2.update.sqlpp | 26 ++ .../groupby_rename_with_sugar.3.query.sqlpp | 25 ++ .../resolution/order_1/order_1.1.ddl.sqlpp | 35 ++ .../resolution/order_1/order_1.2.update.sqlpp | 29 ++ .../resolution/order_1/order_1.3.query.sqlpp | 24 ++ .../select-star/from/from.3.query.sqlpp | 8 +- .../select-star/join/join.3.query.sqlpp | 12 +- .../select-star/mixed/mixed.3.query.sqlpp | 10 +- .../select-star/no_star/no_star.3.query.sqlpp | 8 +- .../overlap_bins_gby_1.3.query.sqlpp | 2 +- .../queries_sqlpp/tpcds/q01/q01.3.query.sqlpp | 28 +- .../queries_sqlpp/tpcds/q03/q03.3.query.sqlpp | 6 +- .../queries_sqlpp/tpcds/q07/q07.3.query.sqlpp | 30 +- .../queries_sqlpp/tpcds/q15/q15.3.query.sqlpp | 28 +- .../queries_sqlpp/tpcds/q19/q19.3.query.sqlpp | 30 +- .../queries_sqlpp/tpcds/q21/q21.3.query.sqlpp | 32 +- .../queries_sqlpp/tpcds/q24a/q24a.3.query.sqlpp | 50 +-- .../queries_sqlpp/tpcds/q24b/q24b.3.query.sqlpp | 50 +-- .../queries_sqlpp/tpcds/q25/q25.3.query.sqlpp | 44 +-- .../queries_sqlpp/tpcds/q26/q26.3.query.sqlpp | 30 +- .../queries_sqlpp/tpcds/q29/q29.3.query.sqlpp | 44 +-- .../queries_sqlpp/tpcds/q30/q30.3.query.sqlpp | 40 +-- .../queries_sqlpp/tpcds/q31/q31.3.query.sqlpp | 20 +- .../queries_sqlpp/tpcds/q34/q34.3.query.sqlpp | 22 +- .../queries_sqlpp/tpcds/q37/q37.3.query.sqlpp | 18 +- .../queries_sqlpp/tpcds/q42/q42.3.query.sqlpp | 4 +- .../queries_sqlpp/tpcds/q43/q43.3.query.sqlpp | 26 +- .../queries_sqlpp/tpcds/q45/q45.3.query.sqlpp | 18 +- .../queries_sqlpp/tpcds/q46/q46.3.query.sqlpp | 32 +- .../queries_sqlpp/tpcds/q50/q50.3.query.sqlpp | 54 +-- .../queries_sqlpp/tpcds/q52/q52.3.query.sqlpp | 4 +- .../queries_sqlpp/tpcds/q55/q55.3.query.sqlpp | 16 +- .../queries_sqlpp/tpcds/q59/q59.3.query.sqlpp | 66 ++-- .../queries_sqlpp/tpcds/q62/q62.3.query.sqlpp | 50 +-- .../queries_sqlpp/tpcds/q68/q68.3.query.sqlpp | 36 +- .../queries_sqlpp/tpcds/q73/q73.3.query.sqlpp | 22 +- .../queries_sqlpp/tpcds/q79/q79.3.query.sqlpp | 12 +- .../queries_sqlpp/tpcds/q81/q81.3.query.sqlpp | 40 +-- .../queries_sqlpp/tpcds/q82/q82.3.query.sqlpp | 18 +- .../queries_sqlpp/tpcds/q85/q85.3.query.sqlpp | 58 ++-- .../queries_sqlpp/tpcds/q91/q91.3.query.sqlpp | 44 +-- .../queries_sqlpp/tpcds/q94/q94.3.query.sqlpp | 24 +- .../queries_sqlpp/tpcds/q95/q95.3.query.sqlpp | 34 +- .../queries_sqlpp/tpcds/q96/q96.3.query.sqlpp | 6 +- .../queries_sqlpp/tpcds/q98/q98.3.query.sqlpp | 44 +-- .../query-ASTERIXDB-1602.3.query.sqlpp | 2 +- .../q13_customer_distribution.3.query.sqlpp | 2 +- ...pliers_who_kept_orders_waiting.3.query.sqlpp | 2 +- .../query-issue601/query-issue601.3.query.sqlpp | 2 +- .../nest_aggregate/nest_aggregate.3.query.sqlpp | 10 +- .../nest_aggregate2.3.query.sqlpp | 10 +- .../q02_minimum_cost_supplier.3.query.sqlpp | 16 +- .../q03_shipping_priority_nt.3.query.sqlpp | 12 +- .../q04_order_priority.3.query.sqlpp | 12 +- .../q05_local_supplier_volume.3.query.sqlpp | 32 +- .../q07_volume_shipping.3.query.sqlpp | 26 +- .../q08_national_market_share.3.query.sqlpp | 52 +-- .../q09_product_type_profit_nt.3.query.sqlpp | 30 +- ..._product_type_profit_parameter.3.query.sqlpp | 30 +- .../q10_returned_item.3.query.sqlpp | 18 +- .../q10_returned_item_int64.3.query.sqlpp | 18 +- .../q11_important_stock.3.query.sqlpp | 32 +- .../q12_shipping/q12_shipping.3.query.sqlpp | 16 +- .../q12_shipping_broadcast.3.query.sqlpp | 16 +- .../q13_customer_distribution.3.query.sqlpp | 4 +- .../q14_promotion_effect.3.query.sqlpp | 10 +- .../q15_top_supplier.3.query.sqlpp | 18 +- ...16_parts_supplier_relationship.3.query.sqlpp | 16 +- ...7_small_quantity_order_revenue.3.query.sqlpp | 12 +- .../q18_large_volume_customer.3.query.sqlpp | 12 +- .../q19_discounted_revenue.3.query.sqlpp | 44 +-- .../q20_potential_part_promotion.3.query.sqlpp | 26 +- ...pliers_who_kept_orders_waiting.3.query.sqlpp | 20 +- .../query-issue562/query-issue562.3.query.sqlpp | 6 +- .../query-issue638/query-issue638.3.query.sqlpp | 24 +- .../query-issue785-2.3.query.sqlpp | 14 +- .../query-issue785/query-issue785.3.query.sqlpp | 12 +- .../query-issue786/query-issue786.3.query.sqlpp | 10 +- .../q13_customer_distribution.3.query.sqlpp | 2 +- ...pliers_who_kept_orders_waiting.3.query.sqlpp | 2 +- .../query-issue601/query-issue601.3.query.sqlpp | 2 +- .../q13_customer_distribution.3.query.sqlpp | 2 +- ...pliers_who_kept_orders_waiting.3.query.sqlpp | 2 +- .../query-issue601/query-issue601.3.query.sqlpp | 2 +- .../union_negative_2.1.ddl.sqlpp | 39 --- .../union_negative_2.2.update.sqlpp | 26 -- .../union_negative_2.3.query.sqlpp | 30 -- .../union_orderby_5.1.ddl.sqlpp | 39 +++ .../union_orderby_5.2.update.sqlpp | 26 ++ .../union_orderby_5.3.query.sqlpp | 30 ++ .../runtimets/results/dapd/q2-7/q2-7.1.adm | 4 +- .../explain_field_access.1.adm | 30 +- .../explain_field_access_closed.1.adm | 26 +- .../query-ASTERIXDB-1577.3.adm | 0 .../single_dataset_with_index.13.adm | 14 +- .../single_dataset_with_index.8.adm | 14 +- .../groupby_rename_with_sugar.3.adm | 3 + .../results/resolution/order_1/order_1.3.adm | 4 + .../union/union_orderby_5/union_orderby_5.1.adm | 35 ++ .../big_object_groupby/big_object_groupby.3.ast | 5 +- .../results_parser_sqlpp/dapd/q2/q2.3.ast | 6 +- .../results_parser_sqlpp/flwor/at06/at06.3.ast | 9 +- .../fuzzyjoin/dblp-1_2.1.1/dblp-1_2.3.ast | 9 +- .../fuzzyjoin/dblp-1_2.1/dblp-1_2.3.ast | 9 +- .../fuzzyjoin/dblp-1_2/dblp-1_2.3.ast | 9 +- .../fuzzyjoin/dblp-2.1_5.3.1/dblp-2.3.ast | 21 +- .../fuzzyjoin/dblp-2.2/dblp-2.3.ast | 6 +- .../fuzzyjoin/dblp-2_2/dblp-2_2.3.ast | 11 +- .../fuzzyjoin/dblp-2_3/dblp-2_3.3.ast | 12 +- .../fuzzyjoin/dblp-2_4/dblp-2_4.3.ast | 13 +- .../fuzzyjoin/dblp-2_5.1/dblp-2_5.3.ast | 14 +- .../fuzzyjoin/dblp-2_5.2/dblp-2_5.3.ast | 14 +- .../fuzzyjoin/dblp-2_5.3.1/dblp-2_5.3.ast | 14 +- .../fuzzyjoin/dblp-2_5.3/dblp-2_5.3.ast | 14 +- .../fuzzyjoin/dblp-2_5/dblp-2_5.3.ast | 14 +- .../fuzzyjoin/dblp-3_1.1/dblp-3_1.3.ast | 53 ++- .../fuzzyjoin/dblp-3_1.2/dblp-3_1.3.ast | 53 ++- .../fuzzyjoin/dblp-3_1/dblp-3_1.3.ast | 55 ++- .../fuzzyjoin/dblp-csx-2_1/dblp-csx-2_1.3.ast | 33 +- .../fuzzyjoin/dblp-csx-2_2/dblp-csx-2_2.3.ast | 62 ++-- .../fuzzyjoin/dblp-csx-2_3/dblp-csx-2_3.3.ast | 67 ++-- .../fuzzyjoin/dblp-csx-2_4/dblp-csx-2_4.3.ast | 72 ++-- .../fuzzyjoin/dblp-csx-2_5.1/dblp-csx-2_5.3.ast | 97 ++---- .../fuzzyjoin/dblp-csx-2_5.2/dblp-csx-2_5.3.ast | 97 ++---- .../dblp-csx-2_5.3.1/dblp-csx-2_5.3.ast | 77 ++--- .../fuzzyjoin/dblp-csx-2_5.3/dblp-csx-2_5.3.ast | 97 ++---- .../fuzzyjoin/dblp-csx-2_5/dblp-csx-2_5.3.ast | 77 ++--- .../fuzzyjoin/dblp-csx-3_1/dblp-csx-3_1.3.ast | 33 +- .../fuzzyjoin/dblp-csx-3_2/dblp-csx-3_2.3.ast | 62 ++-- .../fuzzyjoin/dblp-csx-3_3/dblp-csx-3_3.3.ast | 67 ++-- .../fuzzyjoin/dblp-csx-3_4/dblp-csx-3_4.3.ast | 72 ++-- .../fuzzyjoin/dblp-csx-3_5.1/dblp-csx-3_5.3.ast | 97 ++---- .../fuzzyjoin/dblp-csx-3_5.2/dblp-csx-3_5.3.ast | 97 ++---- .../dblp-csx-3_5.3.1/dblp-csx-3_5.3.ast | 97 ++---- .../fuzzyjoin/dblp-csx-3_5.3/dblp-csx-3_5.3.ast | 77 ++--- .../dblp-csx-3_5.4.1/dblp-csx-3_5.3.ast | 99 ++---- .../fuzzyjoin/dblp-csx-3_5.4/dblp-csx-3_5.3.ast | 99 ++---- .../fuzzyjoin/dblp-csx-3_5/dblp-csx-3_5.3.ast | 77 ++--- .../hdfs/hdfs_02/hdfs_02.3.ast | 5 +- .../hdfs/hdfs_03/hdfs_03.3.ast | 5 +- .../hdfs_shortcircuit/hdfs_shortcircuit.3.ast | 5 +- .../issue_251_dataset_hint_6.3.ast | 5 +- .../groupby-orderby-count.3.ast | 10 +- .../query-issue456/query-issue456.3.ast | 16 +- .../count-nullable/count-nullable.3.ast | 5 +- .../cell-aggregation-with-filtering.3.ast | 7 +- .../cell-aggregation/cell-aggregation.3.ast | 5 +- .../overlap_bins_gby_0/overlap_bins_gby_0.3.ast | 5 +- .../overlap_bins_gby_1/overlap_bins_gby_1.3.ast | 13 +- .../tinysocial-suite.26.ast | 18 +- .../tinysocial-suite.27.ast | 6 + .../q01_pricing_summary_report_nt.3.ast | 6 +- .../q03_shipping_priority_nt.3.ast | 7 +- .../q05_local_supplier_volume.3.ast | 5 +- .../q06_forecast_revenue_change.3.ast | 5 + .../q07_volume_shipping.3.ast | 1 + .../q10_returned_item/q10_returned_item.3.ast | 11 +- .../q10_returned_item_int64.3.ast | 11 +- .../q13_customer_distribution.3.ast | 24 +- .../q14_promotion_effect.3.ast | 18 +- .../q16_parts_supplier_relationship.3.ast | 7 +- .../q17_large_gby_variant.3.ast | 55 ++- .../q18_large_volume_customer.3.ast | 16 +- .../q20_potential_part_promotion.3.ast | 6 +- .../q21_suppliers_who_kept_orders_waiting.3.ast | 19 +- .../q22_global_sales_opportunity.3.ast | 10 +- .../query-issue601/query-issue601.3.ast | 31 +- .../q01_pricing_summary_report_nt.3.ast | 6 +- .../q03_shipping_priority_nt.3.ast | 7 +- .../q05_local_supplier_volume.3.ast | 5 +- .../q06_forecast_revenue_change.4.ast | 5 + .../q07_volume_shipping.3.ast | 1 + .../q08_national_market_share.3.ast | 10 +- .../q09_product_type_profit_nt.3.ast | 6 +- .../tpch/q12_shipping/q12_shipping.3.ast | 10 +- .../q13_customer_distribution.3.ast | 20 +- .../q14_promotion_effect.3.ast | 18 +- .../q16_parts_supplier_relationship.3.ast | 7 +- .../q17_large_gby_variant.3.ast | 55 ++- .../q18_large_volume_customer.3.ast | 16 +- .../q20_potential_part_promotion.3.ast | 6 +- .../q21_suppliers_who_kept_orders_waiting.3.ast | 19 +- .../q22_global_sales_opportunity.3.ast | 10 +- .../tpch/query-issue562/query-issue562.3.ast | 5 +- .../tpch/query-issue601/query-issue601.3.ast | 31 +- .../query-issue810-3/query-issue810-3.3.ast | 30 +- .../user-defined-functions/udf30/udf30.1.ast | 2 +- .../resources/runtimets/testsuite_sqlpp.xml | 25 +- asterixdb/asterix-lang-common/pom.xml | 4 + .../lang/common/base/IReturningStatement.java | 14 + .../asterix/lang/common/context/Scope.java | 83 ++--- .../lang/common/expression/OperatorExpr.java | 2 +- .../lang/common/parser/ScopeChecker.java | 28 +- .../lang/common/statement/InsertStatement.java | 11 + .../asterix/lang/common/statement/Query.java | 18 + .../lang/common/struct/VarIdentifier.java | 22 +- .../visitor/AbstractInlineUdfsVisitor.java | 7 +- asterixdb/asterix-lang-sqlpp/pom.xml | 4 + .../rewrites/SqlppFunctionBodyRewriter.java | 15 +- .../lang/sqlpp/rewrites/SqlppQueryRewriter.java | 36 +- .../SqlppGlobalAggregationSugarVisitor.java | 63 ---- .../SqlppGroupByAggregationSugarVisitor.java | 303 +++++++++++++++++ .../visitor/SqlppGroupBySugarVisitor.java | 141 -------- .../rewrites/visitor/SqlppGroupByVisitor.java | 279 ++++----------- .../visitor/VariableCheckAndRewriteVisitor.java | 158 +++++---- .../lang/sqlpp/util/SqlppRewriteUtil.java | 10 - .../lang/sqlpp/util/SqlppVariableUtil.java | 19 -- .../lang/sqlpp/visitor/DeepCopyVisitor.java | 3 +- .../lang/sqlpp/visitor/FreeVariableVisitor.java | 23 +- .../sqlpp/visitor/SqlppAstPrintVisitor.java | 34 +- ...SqlppCloneAndSubstituteVariablesVisitor.java | 4 +- .../visitor/SqlppDeleteRewriteVisitor.java | 1 - .../AbstractSqlppExpressionScopingVisitor.java | 38 +-- .../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 2 + .../asterix/om/functions/BuiltinFunctions.java | 5 +- .../physical/ExternalGroupByPOperator.java | 10 +- 315 files changed, 3366 insertions(+), 4386 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java index 5146993..bd67ed4 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java @@ -70,7 +70,6 @@ import org.apache.asterix.optimizer.rules.RemoveRedundantListifyRule; import org.apache.asterix.optimizer.rules.RemoveRedundantSelectRule; import org.apache.asterix.optimizer.rules.RemoveSortInFeedIngestionRule; import org.apache.asterix.optimizer.rules.RemoveUnusedOneToOneEquiJoinRule; -import org.apache.asterix.optimizer.rules.ResolveVariableRule; import org.apache.asterix.optimizer.rules.SetAsterixPhysicalOperatorsRule; import org.apache.asterix.optimizer.rules.SetClosedRecordConstructorsRule; import org.apache.asterix.optimizer.rules.SetupCommitExtensionOpRule; @@ -166,7 +165,6 @@ public final class RuleCollections { public static final List<IAlgebraicRewriteRule> buildNormalizationRuleCollection(ICcApplicationContext appCtx) { List<IAlgebraicRewriteRule> normalization = new LinkedList<>(); - normalization.add(new ResolveVariableRule()); normalization.add(new CheckInsertUpsertReturningRule()); normalization.add(new IntroduceUnnestForCollectionToSequenceRule()); normalization.add(new EliminateSubplanRule()); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ByNameToByIndexFieldAccessRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ByNameToByIndexFieldAccessRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ByNameToByIndexFieldAccessRule.java index b3b0ad7..1ace6fa 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ByNameToByIndexFieldAccessRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ByNameToByIndexFieldAccessRule.java @@ -87,7 +87,7 @@ public class ByNameToByIndexFieldAccessRule implements IAlgebraicRewriteRule { return changed; } changed |= extractFirstArg(fce, op, context); - IVariableTypeEnvironment env = context.getOutputTypeEnvironment(op); + IVariableTypeEnvironment env = context.getOutputTypeEnvironment(op.getInputs().get(0).getValue()); IAType t = (IAType) env.getType(fce.getArguments().get(0).getValue()); changed |= rewriteFieldAccess(exprRef, fce, getActualType(t)); return changed; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggregateIntoNestedSubplanRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggregateIntoNestedSubplanRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggregateIntoNestedSubplanRule.java index 2b98762..d69dd3b 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggregateIntoNestedSubplanRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggregateIntoNestedSubplanRule.java @@ -421,47 +421,47 @@ public class PushAggregateIntoNestedSubplanRule implements IAlgebraicRewriteRule AggregateOperator nspAgg = (AggregateOperator) nspAggRef.getValue(); Mutable<ILogicalOperator> nspAggChildRef = nspAgg.getInputs().get(0); LogicalVariable listifyVar = findListifiedVariable(nspAgg, varFromNestedAgg); - if (listifyVar == null) { - continue; - } - OperatorManipulationUtil.substituteVarRec(aggInSubplanOp, unnestVar, listifyVar, true, context); - nspAgg.getVariables().addAll(aggInSubplanOp.getVariables()); - nspAgg.getExpressions().addAll(aggInSubplanOp.getExpressions()); - for (LogicalVariable v : aggInSubplanOp.getVariables()) { - nspWithAgg.put(v, nspOp); - nspAggVars.put(v, 0); - nspAggVarToPlanIndex.put(v, i); - } + if (listifyVar != null) { + OperatorManipulationUtil.substituteVarRec(aggInSubplanOp, unnestVar, listifyVar, true, context); + nspAgg.getVariables().addAll(aggInSubplanOp.getVariables()); + nspAgg.getExpressions().addAll(aggInSubplanOp.getExpressions()); + for (LogicalVariable v : aggInSubplanOp.getVariables()) { + nspWithAgg.put(v, nspOp); + nspAggVars.put(v, 0); + nspAggVarToPlanIndex.put(v, i); + } - Mutable<ILogicalOperator> opRef1InSubplan = aggInSubplanOp.getInputs().get(0); - if (!opRef1InSubplan.getValue().getInputs().isEmpty()) { - Mutable<ILogicalOperator> opRef2InSubplan = opRef1InSubplan.getValue().getInputs().get(0); - AbstractLogicalOperator op2InSubplan = (AbstractLogicalOperator) opRef2InSubplan.getValue(); - if (op2InSubplan.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) { - List<Mutable<ILogicalOperator>> nspInpList = nspAgg.getInputs(); - nspInpList.clear(); - nspInpList.add(opRef1InSubplan); - while (true) { - opRef2InSubplan = opRef1InSubplan.getValue().getInputs().get(0); - op2InSubplan = (AbstractLogicalOperator) opRef2InSubplan.getValue(); - if (op2InSubplan.getOperatorTag() == LogicalOperatorTag.UNNEST) { - List<Mutable<ILogicalOperator>> opInpList = opRef1InSubplan.getValue().getInputs(); - opInpList.clear(); - opInpList.add(nspAggChildRef); - break; - } - opRef1InSubplan = opRef2InSubplan; - if (opRef1InSubplan.getValue().getInputs().isEmpty()) { - throw new IllegalStateException( + Mutable<ILogicalOperator> opRef1InSubplan = aggInSubplanOp.getInputs().get(0); + if (!opRef1InSubplan.getValue().getInputs().isEmpty()) { + Mutable<ILogicalOperator> opRef2InSubplan = opRef1InSubplan.getValue().getInputs().get(0); + AbstractLogicalOperator op2InSubplan = (AbstractLogicalOperator) opRef2InSubplan.getValue(); + if (op2InSubplan.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) { + List<Mutable<ILogicalOperator>> nspInpList = nspAgg.getInputs(); + nspInpList.clear(); + nspInpList.add(opRef1InSubplan); + while (true) { + opRef2InSubplan = opRef1InSubplan.getValue().getInputs().get(0); + op2InSubplan = (AbstractLogicalOperator) opRef2InSubplan.getValue(); + if (op2InSubplan.getOperatorTag() == LogicalOperatorTag.UNNEST) { + List<Mutable<ILogicalOperator>> opInpList = opRef1InSubplan.getValue().getInputs(); + opInpList.clear(); + opInpList.add(nspAggChildRef); + break; + } + opRef1InSubplan = opRef2InSubplan; + if (opRef1InSubplan.getValue().getInputs().isEmpty()) { + throw new IllegalStateException( "PushAggregateIntoNestedSubplanRule: could not find UNNEST."); + } } } } + subplanOpRef.setValue(subplan.getInputs().get(0).getValue()); + OperatorPropertiesUtil.typeOpRec(nspAggRef, context); + return true; } - subplanOpRef.setValue(subplan.getInputs().get(0).getValue()); - OperatorPropertiesUtil.typeOpRec(nspAggRef, context); } - return true; + return false; } private LogicalVariable findListifiedVariable(AggregateOperator nspAgg, LogicalVariable varFromNestedAgg) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java deleted file mode 100644 index 2d7695f..0000000 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.optimizer.rules; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.asterix.lang.common.util.FunctionUtil; -import org.apache.asterix.metadata.declared.MetadataProvider; -import org.apache.asterix.om.base.AString; -import org.apache.asterix.om.constants.AsterixConstantValue; -import org.apache.asterix.om.functions.BuiltinFunctions; -import org.apache.asterix.om.types.ARecordType; -import org.apache.asterix.om.types.ATypeTag; -import org.apache.asterix.om.types.AUnionType; -import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.utils.ConstantExpressionUtil; -import org.apache.commons.lang3.mutable.Mutable; -import org.apache.commons.lang3.mutable.MutableObject; -import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; -import org.apache.hyracks.algebricks.common.utils.Pair; -import org.apache.hyracks.algebricks.common.utils.Triple; -import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; -import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; -import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext; -import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag; -import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; -import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression; -import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; -import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression; -import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; - -/** - * This rule resolves references to undefined identifiers with the following priority: - * 1. field-access - * 2. datasets - * based on the available type and metadata information. - * - * - * Note that undefined variable references that are FROM/JOIN/UNNEST/Quantifier binding expressions - * are resolved to dataset only, which has been done in - * - * @see org.apache.asterix.lang.sqlpp.rewrites.visitor.VariableCheckAndRewriteVisitor - * - */ -public class ResolveVariableRule implements IAlgebraicRewriteRule { - - @Override - public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) - throws AlgebricksException { - return false; - } - - @Override - public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) - throws AlgebricksException { - ILogicalOperator op = opRef.getValue(); - if (op.getInputs().isEmpty()) { - return false; - } - // Populates the latest type information, e.g., resolved path sugars. - context.computeAndSetTypeEnvironmentForOperator(op); - if (op.acceptExpressionTransform( - exprRef -> rewriteExpressionReference(op, exprRef, new Triple<>(false, null, null), null, context))) { - // Generates the up-to-date type information. - context.computeAndSetTypeEnvironmentForOperator(op); - return true; - } - return false; - } - - // Recursively rewrites for an expression within an operator. - private boolean rewriteExpressionReference(ILogicalOperator op, Mutable<ILogicalExpression> exprRef, - Triple<Boolean, String, String> fullyQualifiedDatasetPathCandidateFromParent, - Mutable<ILogicalExpression> parentFuncRef, IOptimizationContext context) throws AlgebricksException { - ILogicalExpression expr = exprRef.getValue(); - if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) { - return false; - } - boolean changed = false; - AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr; - Triple<Boolean, String, String> fullyQualifiedDatasetPathCandidate = resolveFullyQualifiedPath(funcExpr, - context); - for (Mutable<ILogicalExpression> funcArgRef : funcExpr.getArguments()) { - if (rewriteExpressionReference(op, funcArgRef, fullyQualifiedDatasetPathCandidate, exprRef, context)) { - changed = true; - } - } - - // Cleans up extra scan-collections if there is. - if (changed) { - cleanupScanCollectionForDataset(funcExpr); - } - - // Does the actual resolution. - return changed || resolve(op, context, exprRef, fullyQualifiedDatasetPathCandidateFromParent, parentFuncRef); - } - - // Resolves a "resolve" function call expression to a fully qualified variable/field-access path or - // a dataset. - private boolean resolve(ILogicalOperator op, IOptimizationContext context, Mutable<ILogicalExpression> exprRef, - Triple<Boolean, String, String> fullyQualifiedDatasetPathCandidateFromParent, - Mutable<ILogicalExpression> parentFuncRef) throws AlgebricksException { - AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) exprRef.getValue(); - if (funcExpr.getFunctionIdentifier() != BuiltinFunctions.RESOLVE) { - return false; - } - ILogicalExpression arg = funcExpr.getArguments().get(0).getValue(); - String unresolvedVarName = extractConstantString(arg); - return resolveInternal(exprRef, hasMatchedDatasetForVariableName(unresolvedVarName, context), - findCandidatePaths(op, extractExprs(funcExpr.getArguments()), unresolvedVarName, context), - unresolvedVarName, fullyQualifiedDatasetPathCandidateFromParent, parentFuncRef, context); - } - - // Extracts all possible expressions from the arguments of the "resolve" function. - private List<ILogicalExpression> extractExprs(List<Mutable<ILogicalExpression>> args) throws AlgebricksException { - List<ILogicalExpression> exprs = new ArrayList<>(); - // The first arg is is the name of the undefined variable. - for (int index = 1; index < args.size(); ++index) { - ILogicalExpression argExpr = args.get(index).getValue(); - exprs.add(argExpr); - } - return exprs; - } - - // Resolves an undefined name to a dataset or a fully qualified variable/field-access path - // based on the given information of dataset matches and candidate paths. - private boolean resolveInternal(Mutable<ILogicalExpression> funcRef, boolean hasMatchedDataset, - Collection<Pair<ILogicalExpression, List<String>>> varAccessCandidates, String unresolvedVarName, - Triple<Boolean, String, String> fullyQualifiedDatasetPathCandidateFromParent, - Mutable<ILogicalExpression> parentFuncRef, IOptimizationContext context) throws AlgebricksException { - AbstractFunctionCallExpression func = (AbstractFunctionCallExpression) funcRef.getValue(); - int numVarCandidates = varAccessCandidates.size(); - - // The resolution order: 1. field-access 2. datasets (standalone-name or fully-qualified) - if (numVarCandidates > 0) { - if (numVarCandidates == 1) { - resolveAsFieldAccess(funcRef, varAccessCandidates.iterator().next()); - } else { - // More than one possibilities. - throw new AlgebricksException( - "Cannot resolve ambiguous alias reference for undefined identifier " + unresolvedVarName); - } - } else if (hasMatchedDataset) { - // Rewrites the "resolve" function to a "dataset" function and only keep the dataset name argument. - func.setFunctionInfo(FunctionUtil.getFunctionInfo(BuiltinFunctions.DATASET)); - Mutable<ILogicalExpression> datasetNameExpression = func.getArguments().get(0); - func.getArguments().clear(); - func.getArguments().add(datasetNameExpression); - } else if (fullyQualifiedDatasetPathCandidateFromParent.first) { - // Rewrites the parent "field-access" function to a "dataset" function. - AbstractFunctionCallExpression parentFunc = (AbstractFunctionCallExpression) parentFuncRef.getValue(); - parentFunc.setFunctionInfo(FunctionUtil.getFunctionInfo(BuiltinFunctions.DATASET)); - parentFunc.getArguments().clear(); - parentFunc.getArguments().add( - new MutableObject<>(new ConstantExpression( - new AsterixConstantValue(new AString(fullyQualifiedDatasetPathCandidateFromParent.second - + "." + fullyQualifiedDatasetPathCandidateFromParent.third))))); - } else { - MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider(); - // Cannot find any resolution. - throw new AlgebricksException("Cannot find dataset " + unresolvedVarName + " in dataverse " - + metadataProvider.getDefaultDataverseName() + " nor an alias with name " + unresolvedVarName); - } - return true; - } - - // Resolves a "resolve" function call as a field access. - private void resolveAsFieldAccess(Mutable<ILogicalExpression> funcRef, - Pair<ILogicalExpression, List<String>> varAndPath) { - // Rewrites to field-access-by-names. - ILogicalExpression expr = varAndPath.first; - List<String> path = varAndPath.second; - Mutable<ILogicalExpression> firstArgRef = new MutableObject<>(expr); - ILogicalExpression newFunc = null; - for (String fieldName : path) { - List<Mutable<ILogicalExpression>> args = new ArrayList<>(); - args.add(firstArgRef); - args.add(new MutableObject<>(new ConstantExpression(new AsterixConstantValue(new AString(fieldName))))); - newFunc = new ScalarFunctionCallExpression( - FunctionUtil.getFunctionInfo(BuiltinFunctions.FIELD_ACCESS_BY_NAME), args); - firstArgRef = new MutableObject<>(newFunc); - } - funcRef.setValue(newFunc); - } - - // Finds all candidate fully qualified expression/field-access paths. - private Set<Pair<ILogicalExpression, List<String>>> findCandidatePaths(ILogicalOperator op, - Collection<ILogicalExpression> referenceExprs, String unresolvedVarName, IOptimizationContext context) - throws AlgebricksException { - Set<Pair<ILogicalExpression, List<String>>> candidates = new HashSet<>(); - IVariableTypeEnvironment env = context.getOutputTypeEnvironment(op.getInputs().get(0).getValue()); - for (ILogicalExpression referenceExpr : referenceExprs) { - IAType type = (IAType) env.getType(referenceExpr); - candidates.addAll(findCandidatePathsForExpr(unresolvedVarName, type, referenceExpr, new ArrayList<>())); - } - return candidates; - } - - // Recursively finds candidate paths under an expression. - private Set<Pair<ILogicalExpression, List<String>>> findCandidatePathsForExpr(String unresolvedVarName, - IAType pathType, ILogicalExpression expr, List<String> parentPath) throws AlgebricksException { - Set<Pair<ILogicalExpression, List<String>>> varAccessCandidates = new HashSet<>(); - IAType type = pathType; - if (type.getTypeTag() == ATypeTag.UNION) { - type = ((AUnionType) type).getActualType(); - } - ATypeTag tag = type.getTypeTag(); - if (tag == ATypeTag.ANY) { - List<String> path = new ArrayList<>(parentPath); - path.add(unresolvedVarName); - varAccessCandidates.add(new Pair<>(expr, path)); - } - if (tag == ATypeTag.OBJECT) { - ARecordType recordType = (ARecordType) type; - if (recordType.canContainField(unresolvedVarName)) { - // If the field name is possible. - List<String> path = new ArrayList<>(parentPath); - path.add(unresolvedVarName); - varAccessCandidates.add(new Pair<>(expr, path)); - } else { - // Recursively identified possible paths. - String[] fieldNames = recordType.getFieldNames(); - IAType[] fieldTypes = recordType.getFieldTypes(); - for (int index = 0; index < fieldNames.length; ++index) { - List<String> path = new ArrayList<>(parentPath); - path.add(fieldNames[index]); - varAccessCandidates.addAll(findCandidatePathsForExpr(unresolvedVarName, fieldTypes[index], expr, - path)); - } - } - } - return varAccessCandidates; - } - - // Try to resolve the expression like resolve("x").foo as x.foo. - private Triple<Boolean, String, String> resolveFullyQualifiedPath(AbstractFunctionCallExpression funcExpr, - IOptimizationContext context) throws AlgebricksException { - if (!funcExpr.getFunctionIdentifier().equals(BuiltinFunctions.FIELD_ACCESS_BY_NAME)) { - return new Triple<>(false, null, null); - } - List<Mutable<ILogicalExpression>> args = funcExpr.getArguments(); - ILogicalExpression firstExpr = args.get(0).getValue(); - ILogicalExpression secondExpr = args.get(1).getValue(); - if (firstExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) { - return new Triple<>(false, null, null); - } - if (secondExpr.getExpressionTag() != LogicalExpressionTag.CONSTANT) { - return new Triple<>(false, null, null); - } - AbstractFunctionCallExpression firstFuncExpr = (AbstractFunctionCallExpression) firstExpr; - if (!firstFuncExpr.getFunctionIdentifier().equals(BuiltinFunctions.RESOLVE)) { - return new Triple<>(false, null, null); - } - ILogicalExpression dataverseNameExpr = firstFuncExpr.getArguments().get(0).getValue(); - String dataverseName = extractConstantString(dataverseNameExpr); - String datasetName = extractConstantString(secondExpr); - return new Triple<>(hasMatchedDataverseDataset(dataverseName, datasetName, context), dataverseName, - datasetName); - } - - // Checks whether the dataverse name and dataset name matche a dataset. - private boolean hasMatchedDataverseDataset(String dataverseName, String datasetName, IOptimizationContext context) - throws AlgebricksException { - MetadataProvider mdp = (MetadataProvider) context.getMetadataProvider(); - return mdp.findDataset(dataverseName, datasetName) != null; - } - - // Checks whether the name matches a dataset. - private boolean hasMatchedDatasetForVariableName(String varName, IOptimizationContext context) - throws AlgebricksException { - MetadataProvider mdp = (MetadataProvider) context.getMetadataProvider(); - if (mdp.findDataset(mdp.getDefaultDataverseName(), varName) != null) { - return true; - } - if (varName.contains(".")) { - String[] path = varName.split("\\."); - if (path.length != 2) { - return false; - } - if (mdp.findDataset(path[0], path[1]) != null) { - return true; - } - } - return false; - } - - // Cleans up scan collections on top of a "dataset" function call since "dataset" - // is an unnest function. - private void cleanupScanCollectionForDataset(AbstractFunctionCallExpression funcExpr) { - if (funcExpr.getFunctionIdentifier() != BuiltinFunctions.SCAN_COLLECTION) { - return; - } - ILogicalExpression arg = funcExpr.getArguments().get(0).getValue(); - if (arg.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) { - return; - } - AbstractFunctionCallExpression argFuncExpr = (AbstractFunctionCallExpression) arg; - if (argFuncExpr.getFunctionIdentifier() != BuiltinFunctions.DATASET) { - return; - } - funcExpr.setFunctionInfo(argFuncExpr.getFunctionInfo()); - funcExpr.getArguments().clear(); - funcExpr.getArguments().addAll(argFuncExpr.getArguments()); - } - - // Extracts the name of an undefined variable. - private String extractConstantString(ILogicalExpression arg) throws AlgebricksException { - final String str = ConstantExpressionUtil.getStringConstant(arg); - if (str == null) { - throw new AlgebricksException("The argument is expected to be a string constant value."); - } - return str; - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java index 719824b..2b83158 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java @@ -173,8 +173,7 @@ class LangExpressionToPlanTranslator public LangExpressionToPlanTranslator(MetadataProvider metadataProvider, int currentVarCounterValue) throws AlgebricksException { - this.context = new TranslationContext(new Counter(currentVarCounterValue)); - this.metadataProvider = metadataProvider; + this(metadataProvider, new Counter(currentVarCounterValue)); } // Keeps the given Counter if one is provided instead of a value. @@ -794,6 +793,8 @@ class LangExpressionToPlanTranslator public Pair<ILogicalOperator, LogicalVariable> visit(GroupbyClause gc, Mutable<ILogicalOperator> tupSource) throws CompilationException { Mutable<ILogicalOperator> topOp = tupSource; + + LogicalVariable groupRecordVar = null; if (gc.hasGroupVar()) { List<Pair<Expression, Identifier>> groupFieldList = gc.getGroupFieldList(); List<Mutable<ILogicalExpression>> groupRecordConstructorArgList = new ArrayList<>(); @@ -805,13 +806,14 @@ class LangExpressionToPlanTranslator ILogicalExpression groupFieldExpr = langExprToAlgExpression(groupField.first, topOp).first; groupRecordConstructorArgList.add(new MutableObject<>(groupFieldExpr)); } - LogicalVariable groupVar = context.newVarFromExpression(gc.getGroupVar()); - AssignOperator groupVarAssignOp = new AssignOperator(groupVar, - new MutableObject<>(new ScalarFunctionCallExpression( - FunctionUtil.getFunctionInfo(BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR), - groupRecordConstructorArgList))); - groupVarAssignOp.getInputs().add(topOp); - topOp = new MutableObject<>(groupVarAssignOp); + MutableObject<ILogicalExpression> groupRecordConstr = new MutableObject<>(new ScalarFunctionCallExpression( + FunctionUtil.getFunctionInfo(BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR), + groupRecordConstructorArgList)); + + groupRecordVar = context.newVar(); + AssignOperator groupRecordVarAssignOp = new AssignOperator(groupRecordVar, groupRecordConstr); + groupRecordVarAssignOp.getInputs().add(topOp); + topOp = new MutableObject<>(groupRecordVarAssignOp); } GroupByOperator gOp = new GroupByOperator(); @@ -831,30 +833,46 @@ class LangExpressionToPlanTranslator } gOp.getInputs().add(topOp); - for (Entry<Expression, VariableExpr> entry : gc.getWithVarMap().entrySet()) { - Pair<ILogicalExpression, Mutable<ILogicalOperator>> listifyInput = langExprToAlgExpression(entry.getKey(), + + if (gc.hasGroupVar()) { + VariableExpr groupVar = gc.getGroupVar(); + LogicalVariable groupLogicalVar = context.newVar(); + ILogicalPlan nestedPlan = createNestedPlanWithAggregate(groupLogicalVar, + BuiltinFunctions.LISTIFY, new VariableReferenceExpression(groupRecordVar), new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(gOp)))); - List<Mutable<ILogicalExpression>> flArgs = new ArrayList<>(1); - flArgs.add(new MutableObject<>(listifyInput.first)); - AggregateFunctionCallExpression fListify = - BuiltinFunctions.makeAggregateFunctionExpression(BuiltinFunctions.LISTIFY, flArgs); - LogicalVariable aggVar = context.newVar(); - AggregateOperator agg = new AggregateOperator(mkSingletonArrayList(aggVar), - mkSingletonArrayList(new MutableObject<>(fListify))); - - agg.getInputs().add(listifyInput.second); - - ILogicalPlan plan = new ALogicalPlanImpl(new MutableObject<>(agg)); - gOp.getNestedPlans().add(plan); - // Hide the variable that was part of the "with", replacing it with - // the one bound by the aggregation op. - context.setVar(entry.getValue(), aggVar); + gOp.getNestedPlans().add(nestedPlan); + context.setVar(groupVar, groupLogicalVar); + } + + if (gc.hasWithMap()) { + for (Entry<Expression, VariableExpr> entry : gc.getWithVarMap().entrySet()) { + VariableExpr withVar = entry.getValue(); + Expression withExpr = entry.getKey(); + Pair<ILogicalExpression, Mutable<ILogicalOperator>> listifyInput = langExprToAlgExpression(withExpr, + new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(gOp)))); + LogicalVariable withLogicalVar = context.newVar(); + ILogicalPlan nestedPlan = createNestedPlanWithAggregate(withLogicalVar, + BuiltinFunctions.LISTIFY, listifyInput.first, listifyInput.second); + gOp.getNestedPlans().add(nestedPlan); + context.setVar(withVar, withLogicalVar); + } } + gOp.setGroupAll(gc.isGroupAll()); gOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, gc.hasHashGroupByHint()); return new Pair<>(gOp, null); } + private ILogicalPlan createNestedPlanWithAggregate(LogicalVariable aggOutputVar, FunctionIdentifier aggFunc, + ILogicalExpression aggFnInput, Mutable<ILogicalOperator> aggOpInput) { + AggregateFunctionCallExpression aggFnCall = BuiltinFunctions.makeAggregateFunctionExpression(aggFunc, + mkSingletonArrayList(new MutableObject<>(aggFnInput))); + AggregateOperator aggOp = new AggregateOperator(mkSingletonArrayList(aggOutputVar), + mkSingletonArrayList(new MutableObject<>(aggFnCall))); + aggOp.getInputs().add(aggOpInput); + return new ALogicalPlanImpl(new MutableObject<>(aggOp)); + } + @Override public Pair<ILogicalOperator, LogicalVariable> visit(IfExpr ifexpr, Mutable<ILogicalOperator> tupSource) throws CompilationException { @@ -1270,8 +1288,11 @@ class LangExpressionToPlanTranslator Mutable<ILogicalOperator> topOpRef) throws CompilationException { switch (expr.getKind()) { case VARIABLE_EXPRESSION: - VariableReferenceExpression ve = - new VariableReferenceExpression(context.getVar(((VariableExpr) expr).getVar().getId())); + LogicalVariable var = context.getVar(((VariableExpr) expr).getVar().getId()); + if (var == null) { + throw new IllegalStateException(String.valueOf(expr)); + } + VariableReferenceExpression ve = new VariableReferenceExpression(var); return new Pair<>(ve, topOpRef); case LITERAL_EXPRESSION: LiteralExpr val = (LiteralExpr) expr; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java index c17b55b..36183c6 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java @@ -662,9 +662,11 @@ class SqlppExpressionToPlanTranslator extends LangExpressionToPlanTranslator imp for (GbyVariableExpressionPair pair : groupbyClause.getGbyPairList()) { fieldBindings.add(getFieldBinding(pair.getVar())); } - if (groupbyClause.hasWithMap() && groupbyClause.hasGroupVar()) { - // Makes sure that we add the re-mapped group variable which refers to a collection. - fieldBindings.add(getFieldBinding(groupbyClause.getWithVarMap().get(groupbyClause.getGroupVar()))); + if (groupbyClause.hasGroupVar()) { + fieldBindings.add(getFieldBinding(groupbyClause.getGroupVar())); + } + if (groupbyClause.hasWithMap()) { + throw new IllegalStateException(groupbyClause.getWithVarMap().values().toString()); // no WITH in SQLPP } return fieldBindings; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java index d78cd94..85b8c8a 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java @@ -192,9 +192,13 @@ public class ParserTestExecutor extends TestExecutor { + "org.apache.asterix.lang.common.rewrites.LangRewritingContext)", declaredFunctions, topExpr, metadataProvider, context); PA.invokeMethod(rewriter, "inlineColumnAlias()"); - PA.invokeMethod(rewriter, "rewriteGlobalAggregations()"); + PA.invokeMethod(rewriter, "generateColumnNames()"); + PA.invokeMethod(rewriter, "substituteGroupbyKeyExpression()"); PA.invokeMethod(rewriter, "rewriteGroupBys()"); - PA.invokeMethod(rewriter, "variableCheckAndRewrite(boolean)", Boolean.TRUE); + PA.invokeMethod(rewriter, "rewriteSetOperations()"); + PA.invokeMethod(rewriter, "variableCheckAndRewrite()"); + PA.invokeMethod(rewriter, "rewriteGroupByAggregationSugar()"); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/denorm-cust-order.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/denorm-cust-order.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/denorm-cust-order.sqlpp index 52ce35c..508d6ec 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/denorm-cust-order.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/denorm-cust-order.sqlpp @@ -62,9 +62,9 @@ create dataset Customers(CustomerType) primary key cid on group1; create dataset Orders(OrderType) primary key oid on group1; write output to asterix_nc1:"/tmp/custorder.adm"; -select element {'cid':cid,'cust':cust,'cnt-orders':count(o),'orders':o} +select element {'cid':cid,'cust':(from g select c),'cnt-orders':count(o),'orders': (from g select o)} from Customers as c, Orders as o where (c.cid = o.cid) -group by c.cid as cid +group by c.cid as cid group as g ; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/orderby-desc-using-gby.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/orderby-desc-using-gby.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/orderby-desc-using-gby.sqlpp index 279adf6..2313e90 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/orderby-desc-using-gby.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/orderby-desc-using-gby.sqlpp @@ -52,6 +52,6 @@ create dataset Customers(CustomerType) primary key cid on group1; write output to asterix_nc1:"rttest/gby-using-orderby-desc.adm"; select element {'name':name,'age':age} from Customers as c -group by c.name as name +group by c.name as name, c.age as age order by name desc,age ; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/orders-aggreg.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/orders-aggreg.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/orders-aggreg.sqlpp index f82ce53..cca0211 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/orders-aggreg.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/orders-aggreg.sqlpp @@ -40,13 +40,13 @@ create nodegroup group1 if not exists on create dataset Orders(OrderType) primary key oid on group1; write output to asterix_nc1:"/tmp/orders-aggreg.adm"; -select element {'cid':cid,'ordpercust':`orders-aggreg`.coll_count(o),'totalcust':`orders-aggreg`.coll_sum(( - select element i.total - from o as i +select element {'cid':cid,'ordpercust':`orders-aggreg`.coll_count(g),'totalcust':`orders-aggreg`.coll_sum(( + select element i.o.total + from g as i )),'avgcust':`orders-aggreg`.coll_avg(( - select element i.total - from o as i + select element i.o.total + from g as i ))} from Orders as o -group by o.cid as cid +group by o.cid as cid group as g ; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/q01_pricing_summary_report_nt.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/q01_pricing_summary_report_nt.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/q01_pricing_summary_report_nt.sqlpp index 2cc5d36..6497eea 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/q01_pricing_summary_report_nt.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/q01_pricing_summary_report_nt.sqlpp @@ -49,30 +49,30 @@ load dataset LineItem using localfs ((`path`=`asterix_nc1://data/tpch0.001/line write output to asterix_nc1:"rttest/tpch_q1_pricing_summary_report_nt.adm"; select element {'l_returnflag':l_returnflag,'l_linestatus':l_linestatus,'sum_qty':tpch.coll_sum(( - select element i.l_quantity - from l as i + select element i.l.l_quantity + from g as i )),'sum_base_price':tpch.coll_sum(( - select element i.l_extendedprice - from l as i + select element i.l.l_extendedprice + from g as i )),'sum_disc_price':tpch.coll_sum(( - select element (i.l_extendedprice * (1 - i.l_discount)) - from l as i + select element (i.l.l_extendedprice * (1 - i.l.l_discount)) + from g as i )),'sum_charge':tpch.coll_sum(( - select element (i.l_extendedprice * (1 - i.l_discount) * (1 + i.l_tax)) - from l as i + select element (i.l.l_extendedprice * (1 - i.l.l_discount) * (1 + i.l.l_tax)) + from g as i )),'ave_qty':tpch.coll_avg(( - select element i.l_quantity - from l as i + select element i.l.l_quantity + from g as i )),'ave_price':tpch.coll_avg(( - select element i.l_extendedprice - from l as i + select element i.l.l_extendedprice + from g as i )),'ave_disc':tpch.coll_avg(( - select element i.l_discount - from l as i - )),'count_order':tpch.coll_count(l)} + select element i.l.l_discount + from g as i + )),'count_order':tpch.coll_count(g)} from LineItem as l where (l.l_shipdate <= '1998-09-02') /* +hash */ -group by l.l_returnflag as l_returnflag,l.l_linestatus as l_linestatus +group by l.l_returnflag as l_returnflag,l.l_linestatus as l_linestatus group as g order by l_returnflag,l_linestatus ; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/query-issue562.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/query-issue562.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/query-issue562.sqlpp index 645d320..7b49736 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/query-issue562.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/query-issue562.sqlpp @@ -92,9 +92,9 @@ with avg as tpch.coll_avg(( with phone_substr as tpch.substring(c.c_phone,1,2) where ((c.c_acctbal > 0.0) and ((phone_substr = '13') or (phone_substr = '31') or (phone_substr = '23') or (phone_substr = '29') or (phone_substr = '30') or (phone_substr = '18') or (phone_substr = '17'))) )) -select element {'cntrycode':cntrycode,'numcust':tpch.coll_count(ct),'totacctbal':tpch.coll_sum(( - select element i.c_acctbal - from ct as i +select element {'cntrycode':cntrycode,'numcust':tpch.coll_count(g),'totacctbal':tpch.coll_sum(( + select element i.ct.c_acctbal + from g as i ))} from tpch.q22_customer_tmp() as ct where (tpch.coll_count(( @@ -102,6 +102,6 @@ where (tpch.coll_count(( from Orders as o where (ct.c_custkey = o.o_custkey) )) = 0) -group by ct.cntrycode as cntrycode +group by ct.cntrycode as cntrycode group as g order by cntrycode ; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/split-materialization-above-join.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/split-materialization-above-join.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/split-materialization-above-join.sqlpp index 187e6cb..2f75c10 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/split-materialization-above-join.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/split-materialization-above-join.sqlpp @@ -40,7 +40,7 @@ select element {'left':paperLeft,'right':paperRight,'sim':ridpair.sim} from DBLP as paperLeft, DBLP as paperRight, ( - select element {'idLeft':idLeft,'idRight':idRight,'sim':sim[0]} + select element {'idLeft':idLeft,'idRight':idRight,'sim':g[0].sim[0]} from DBLP as paperLeft, fuzzyjoin.`subset-collection`(tokensLeft,0,fuzzyjoin.`prefix-len-jaccard`(lenLeft,0.500000f)) as prefixTokenLeft, DBLP as paperRight, @@ -75,7 +75,7 @@ from DBLP as paperLeft, ), sim as fuzzyjoin.`similarity-jaccard-prefix`(lenLeft,tokensLeft,lenRight,tokensRight,prefixTokenLeft,0.500000f) where ((prefixTokenLeft = prefixTokenRight) and ((sim >= 0.500000f) and (paperLeft.id < paperRight.id))) - group by paperLeft.id as idLeft,paperRight.id as idRight + group by paperLeft.id as idLeft,paperRight.id as idRight group as g(sim as sim) ) as ridpair where ((ridpair.idLeft = paperLeft.id) and (ridpair.idRight = paperRight.id)) order by paperLeft.id,paperRight.id http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/constant-gby-agg.plan ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/constant-gby-agg.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/constant-gby-agg.plan index 26601ba..c83a85c 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/constant-gby-agg.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/constant-gby-agg.plan @@ -3,13 +3,13 @@ -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- SORT_GROUP_BY[$$18] |PARTITIONED| + -- SORT_GROUP_BY[$$38] |PARTITIONED| { -- AGGREGATE |LOCAL| -- NESTED_TUPLE_SOURCE |LOCAL| } - -- HASH_PARTITION_EXCHANGE [$$18] |PARTITIONED| - -- SORT_GROUP_BY[$$14] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$38] |PARTITIONED| + -- SORT_GROUP_BY[$$34] |PARTITIONED| { -- AGGREGATE |LOCAL| -- NESTED_TUPLE_SOURCE |LOCAL| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/resources/optimizerts/results/gby_partitioning_property_01.plan ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/gby_partitioning_property_01.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/gby_partitioning_property_01.plan index 7e11745..0d7d538 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/gby_partitioning_property_01.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/gby_partitioning_property_01.plan @@ -3,13 +3,13 @@ -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- PRE_CLUSTERED_GROUP_BY[$$32] |PARTITIONED| + -- PRE_CLUSTERED_GROUP_BY[$$59] |PARTITIONED| { -- AGGREGATE |LOCAL| -- NESTED_TUPLE_SOURCE |LOCAL| } -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- SORT_GROUP_BY[$$22] |PARTITIONED| + -- SORT_GROUP_BY[$$49] |PARTITIONED| { -- AGGREGATE |LOCAL| -- NESTED_TUPLE_SOURCE |LOCAL| @@ -17,8 +17,8 @@ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$22][$$25] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$22] |PARTITIONED| + -- HYBRID_HASH_JOIN [$$49][$$52] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$49] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| @@ -26,7 +26,7 @@ -- DATASOURCE_SCAN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$25] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$52] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_1.plan ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_1.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_1.plan index d45ccfe..86bb1c8 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_1.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_1.plan @@ -4,7 +4,7 @@ -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$21, $$23, $$25][$$22, $$24, $$26] |PARTITIONED| + -- HYBRID_HASH_JOIN [$$35, $$37, $$39][$$36, $$38, $$40] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| @@ -20,4 +20,4 @@ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- DATASOURCE_SCAN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| \ No newline at end of file + -- EMPTY_TUPLE_SOURCE |PARTITIONED| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_2.plan ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_2.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_2.plan index d45ccfe..86bb1c8 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_2.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_2.plan @@ -4,7 +4,7 @@ -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$21, $$23, $$25][$$22, $$24, $$26] |PARTITIONED| + -- HYBRID_HASH_JOIN [$$35, $$37, $$39][$$36, $$38, $$40] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| @@ -20,4 +20,4 @@ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- DATASOURCE_SCAN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| \ No newline at end of file + -- EMPTY_TUPLE_SOURCE |PARTITIONED| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_3.plan ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_3.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_3.plan index b7669dd..240b1aa 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_3.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_3.plan @@ -4,8 +4,8 @@ -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$21, $$24, $$25][$$22, $$23, $$26] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$21, $$24, $$25] |PARTITIONED| + -- HYBRID_HASH_JOIN [$$35, $$38, $$39][$$36, $$37, $$40] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$35, $$38, $$39] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| @@ -13,11 +13,11 @@ -- DATASOURCE_SCAN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$22, $$23, $$26] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$36, $$37, $$40] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- DATASOURCE_SCAN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| \ No newline at end of file + -- EMPTY_TUPLE_SOURCE |PARTITIONED| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/resources/optimizerts/results/loj-core.plan ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/loj-core.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/loj-core.plan index 1d34c95..2cab787 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/loj-core.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/loj-core.plan @@ -2,20 +2,20 @@ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| - -- SORT_MERGE_EXCHANGE [$$15(ASC) ] |PARTITIONED| - -- STABLE_SORT [$$15(ASC)] |PARTITIONED| + -- SORT_MERGE_EXCHANGE [$$36(ASC) ] |PARTITIONED| + -- STABLE_SORT [$$36(ASC)] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$17][$$18] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$17] |PARTITIONED| + -- HYBRID_HASH_JOIN [$$38][$$39] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$38] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- DATASOURCE_SCAN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$18] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$39] |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/resources/optimizerts/results/loj-sugar.plan ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/loj-sugar.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/loj-sugar.plan index 1d34c95..a547407 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/loj-sugar.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/loj-sugar.plan @@ -2,20 +2,20 @@ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| - -- SORT_MERGE_EXCHANGE [$$15(ASC) ] |PARTITIONED| - -- STABLE_SORT [$$15(ASC)] |PARTITIONED| + -- SORT_MERGE_EXCHANGE [$$29(ASC) ] |PARTITIONED| + -- STABLE_SORT [$$29(ASC)] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$17][$$18] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$17] |PARTITIONED| + -- HYBRID_HASH_JOIN [$$31][$$32] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$31] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- DATASOURCE_SCAN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$18] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$32] |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/resources/optimizerts/results/open-index-non-enforced/btree-equi-join-non-enforced/btree-equi-join-non-enforced-05.plan ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/open-index-non-enforced/btree-equi-join-non-enforced/btree-equi-join-non-enforced-05.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/open-index-non-enforced/btree-equi-join-non-enforced/btree-equi-join-non-enforced-05.plan index 15f9171..7becb2c 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/open-index-non-enforced/btree-equi-join-non-enforced/btree-equi-join-non-enforced-05.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/open-index-non-enforced/btree-equi-join-non-enforced/btree-equi-join-non-enforced-05.plan @@ -2,13 +2,13 @@ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| - -- SORT_MERGE_EXCHANGE [$$18(ASC), $$19(ASC) ] |PARTITIONED| - -- STABLE_SORT [$$18(ASC), $$19(ASC)] |PARTITIONED| + -- SORT_MERGE_EXCHANGE [$$32(ASC), $$33(ASC) ] |PARTITIONED| + -- STABLE_SORT [$$32(ASC), $$33(ASC)] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$16][$$17] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$16] |PARTITIONED| + -- HYBRID_HASH_JOIN [$$30][$$31] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$30] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| @@ -16,7 +16,7 @@ -- DATASOURCE_SCAN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$17] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$31] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/resources/optimizerts/results/open-index-non-enforced/btree-equi-join-non-enforced/btree-equi-join-non-enforced-06.plan ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/open-index-non-enforced/btree-equi-join-non-enforced/btree-equi-join-non-enforced-06.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/open-index-non-enforced/btree-equi-join-non-enforced/btree-equi-join-non-enforced-06.plan index 15f9171..7becb2c 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/open-index-non-enforced/btree-equi-join-non-enforced/btree-equi-join-non-enforced-06.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/open-index-non-enforced/btree-equi-join-non-enforced/btree-equi-join-non-enforced-06.plan @@ -2,13 +2,13 @@ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| - -- SORT_MERGE_EXCHANGE [$$18(ASC), $$19(ASC) ] |PARTITIONED| - -- STABLE_SORT [$$18(ASC), $$19(ASC)] |PARTITIONED| + -- SORT_MERGE_EXCHANGE [$$32(ASC), $$33(ASC) ] |PARTITIONED| + -- STABLE_SORT [$$32(ASC), $$33(ASC)] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$16][$$17] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$16] |PARTITIONED| + -- HYBRID_HASH_JOIN [$$30][$$31] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$30] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| @@ -16,7 +16,7 @@ -- DATASOURCE_SCAN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$17] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$31] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-app/src/test/resources/optimizerts/results/open-index-non-enforced/btree-equi-join-non-enforced/btree-equi-join-non-enforced-07.plan ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/open-index-non-enforced/btree-equi-join-non-enforced/btree-equi-join-non-enforced-07.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/open-index-non-enforced/btree-equi-join-non-enforced/btree-equi-join-non-enforced-07.plan index 7f5a841..913c442 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/open-index-non-enforced/btree-equi-join-non-enforced/btree-equi-join-non-enforced-07.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/open-index-non-enforced/btree-equi-join-non-enforced/btree-equi-join-non-enforced-07.plan @@ -2,13 +2,13 @@ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| - -- SORT_MERGE_EXCHANGE [$$20(ASC), $$21(ASC) ] |PARTITIONED| - -- STABLE_SORT [$$20(ASC), $$21(ASC)] |PARTITIONED| + -- SORT_MERGE_EXCHANGE [$$34(ASC), $$35(ASC) ] |PARTITIONED| + -- STABLE_SORT [$$34(ASC), $$35(ASC)] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$17][$$18] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$17] |PARTITIONED| + -- HYBRID_HASH_JOIN [$$31][$$32] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$31] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| @@ -16,7 +16,7 @@ -- DATASOURCE_SCAN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$18] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$32] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED|
