This is an automated email from the ASF dual-hosted git repository. alsuliman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit efb667d4ce2bb3f869840d686131db5a51ed4f9f Merge: 95ad52066b 5316f0ce23 Author: Ali Alsuliman <[email protected]> AuthorDate: Sun Feb 26 17:44:34 2023 -0800 Merge branch 'neo' into master Change-Id: If34921e98a56e3154833eb5958427663c69f3e79 .../provider/SqlppCompilationProvider.java | 15 +- .../asterix/optimizer/base/RuleCollections.java | 6 + .../rules/AbstractConditionExpressionRule.java | 86 ++ .../rules/ExtractRedundantVariablesInJoinRule.java | 187 +++ ...neAndRemoveRedundantBooleanExpressionsRule.java | 107 ++ ...emoveRedundantBooleanExpressionsInJoinRule.java | 155 ++ .../optimizer/rules/am/AccessMethodUtils.java | 6 +- .../optimizer/rules/am/BTreeAccessMethod.java | 1 + .../rules/am/IntroduceSelectAccessMethodRule.java | 25 +- .../array/AbstractOperatorFromSubplanRewrite.java | 46 +- .../rules/am/array/JoinFromSubplanRewrite.java | 16 +- .../rules/am/array/SelectFromSubplanRewrite.java | 139 +- .../optimizer/rules/cbo/EnumerateJoinsRule.java | 136 +- .../rules/cbo/EstimatedCostComputationVisitor.java | 12 +- .../asterix/optimizer/rules/cbo/JoinNode.java | 99 +- .../asterix/optimizer/rules/cbo/PlanNode.java | 40 + .../SqlppExpressionToPlanTranslator.java | 11 + .../apache/asterix/api/common/APIFramework.java | 19 +- .../asterix/app/function/DatasetRewriter.java | 13 + .../asterix/app/function/DumpIndexReader.java | 59 +- .../asterix/app/function/FunctionRewriter.java | 7 +- .../asterix/app/function/QueryIndexDatasource.java | 131 ++ .../asterix/app/function/QueryIndexRewriter.java | 211 +++ .../asterix/util/MetadataBuiltinFunctions.java | 5 + .../asterix/test/runtime/MetricsExecutionTest.java | 2 +- .../apache/asterix/test/runtime/RebalanceTest.java | 2 +- .../test/runtime/SqlppNumericIndexRQGTest.java | 3 +- .../hash-join-with-redundant-variable.1.sqlpp | 63 + .../hash-join-with-redundant-variable.2.sqlpp} | 14 +- .../hash-join-with-redundant-variable.3.sqlpp} | 16 +- .../hash-join-with-redundant-variable.4.sqlpp} | 8 +- .../atomic-and-array-queries/query8.plan | 11 +- .../atomic-and-array-queries/query9.plan | 11 +- .../hash-join-with-redundant-variable.1.plan | 74 + .../hash-join-with-redundant-variable.2.plan | 35 + .../hash-join-with-redundant-variable.3.plan | 35 + .../hash-join-with-redundant-variable.4.plan | 24 + .../atomic-and-array-queries/query1.plan | 14 + .../atomic-and-array-queries/query2.plan | 14 + .../atomic-and-array-queries/query3.plan | 18 + .../atomic-and-array-queries/query8.plan | 11 +- .../atomic-and-array-queries/query9.plan | 11 +- .../closed/use-case-1/query1.plan | 14 + .../closed/use-case-1/query2.plan | 17 + .../closed/use-case-2/query1.plan | 14 + .../closed/use-case-2/query2.plan | 17 + .../closed/use-case-3/query1.plan | 14 + .../closed/use-case-3/query2.plan | 19 + .../closed/use-case-3/query3.plan | 14 + .../closed/use-case-4/query1.plan | 18 + .../closed/use-case-4/query2.plan | 21 + .../closed/with-3-level-record-path/query1.plan | 14 + .../closed/with-3-level-record-path/query2.plan | 17 + .../closed/with-composite-pk/query1.plan | 15 + .../closed/with-composite-pk/query2.plan | 17 + .../closed/with-composite-sk/query1.plan | 16 + .../closed/with-composite-sk/query2.plan | 19 + .../closed/with-filter-fields/query1.plan | 17 + .../closed/with-filter-fields/query2.plan | 17 + .../open/complex-structures/query1.plan | 13 + .../open/complex-structures/query2.plan | 13 + .../open/complex-structures/query3.plan | 16 + .../open/complex-structures/query4.plan | 16 + .../open/complex-structures/query5.plan | 13 + .../open/complex-structures/query6.plan | 13 + .../open/complex-structures/query7.plan | 16 + .../open/complex-structures/query8.plan | 16 + .../open/multiple-indexes/query1.plan | 13 + .../open/multiple-indexes/query2.plan | 13 + .../open/use-case-1/query1.plan | 14 + .../open/use-case-1/query2.plan | 17 + .../open/use-case-2/query1.plan | 14 + .../open/use-case-2/query2.plan | 17 + .../open/use-case-3/query1.plan | 14 + .../open/use-case-3/query2.plan | 19 + .../open/use-case-3/query3.plan | 14 + .../open/use-case-4/query1.plan | 18 + .../open/use-case-4/query2.plan | 21 + .../open/with-3-level-record-path/query1.plan | 14 + .../open/with-3-level-record-path/query2.plan | 17 + .../open/with-composite-sk/query1.plan | 16 + .../btree-index-composite-key-04.plan | 12 + .../cast-default-null/cast-default-null-02.plan | 12 + .../cast-default-null/cast-default-null-10.plan | 12 + .../cast-default-null/cast-default-null-11.plan | 12 + .../cast-default-null/cast-default-null-12.plan | 12 + .../cast-default-null/cast-default-null-13.plan | 12 + .../cast-default-null/cast-default-null-14.plan | 12 + .../cast-default-null/cast-default-null-15.plan | 12 + .../cast-default-null/cast-default-null-20.plan | 12 + .../cast-default-null/cast-default-null-23.plan | 12 + .../cast-default-null/cast-default-null-24.plan | 12 + .../cast-default-null/cast-default-null-25.plan | 12 + .../hints-skip-index/hints-skip-index-10.plan | 12 + .../hints-skip-index/hints-skip-index-13.plan | 12 + .../hints-skip-index/hints-skip-index-4.plan | 12 + .../hints-skip-index/hints-skip-index-6.plan | 12 + .../hints-skip-index/hints-skip-index-7.plan | 12 + .../hints-skip-index/hints-skip-index-8.plan | 12 + .../hints-skip-index/hints-skip-index-9.plan | 12 + .../hints-use-index/hints-use-index-10.plan | 12 + .../hints-use-index/hints-use-index-11.plan | 12 + .../hints-use-index/hints-use-index-12.plan | 12 + .../hints-use-index/hints-use-index-13.plan | 12 + .../hints-use-index/hints-use-index-14.plan | 12 + .../hints-use-index/hints-use-index-15.plan | 12 + .../hints-use-index/hints-use-index-16.plan | 12 + .../hints-use-index/hints-use-index-17.plan | 12 + .../hints-use-index/hints-use-index-18.plan | 12 + .../hints-use-index/hints-use-index-19.plan | 12 + .../hints-use-index/hints-use-index-3.plan | 12 + .../hints-use-index/hints-use-index-4.plan | 12 + .../hints-use-index/hints-use-index-5.plan | 12 + .../hints-use-index/hints-use-index-6.plan | 12 + .../hints-use-index/hints-use-index-7.plan | 12 + .../hints-use-index/hints-use-index-8.plan | 12 + .../hints-use-index/hints-use-index-9.plan | 12 + .../btree-index/btree-composite-key-03.plan | 8 + .../btree-index/btree-secondary-33.plan | 8 + .../btree-index/btree-secondary-34.plan | 8 + .../btree-index/btree-secondary-35.plan | 8 + .../btree-index/btree-secondary-36.plan | 8 + .../btree-index/btree-secondary-40.plan | 8 + .../btree-index/btree-secondary-42.plan | 8 + .../btree-index/btree-secondary-43.plan | 8 + .../btree-index/btree-secondary-44.plan | 8 + .../btree-index/btree-secondary-45.plan | 8 + .../btree-index/btree-secondary-46.plan | 8 + .../btree-index/btree-secondary-47.plan | 10 + .../btree-index/btree-secondary-48.plan | 10 + .../btree-index/btree-secondary-49.plan | 8 + .../btree-index/btree-secondary-51.plan | 10 + .../btree-index/btree-secondary-52.plan | 10 + .../btree-index/btree-secondary-53.plan | 10 + .../btree-index/btree-secondary-54.plan | 8 + .../btree-index/btree-secondary-55.plan | 8 + .../btree-index/btree-secondary-56.plan | 8 + .../btree-index/btree-secondary-57.plan | 8 + .../btree-index/btree-secondary-58.plan | 10 + .../btree-index/btree-secondary-59.plan | 10 + .../btree-index/btree-secondary-60.plan | 8 + .../btree-index/btree-secondary-61.plan | 10 + .../btree-index/btree-secondary-62.plan | 8 + .../btree-index/btree-secondary-63.plan | 8 + .../btree-index/btree-secondary-68.plan | 14 + .../btree-index/btree-secondary-68_ps.plan | 33 + .../btree-sidx-idxonly-01-disable-idxonly.plan | 12 + .../btree-sidx-idxonly-01-disable-idxonly_ps.plan | 33 + .../btree-index/btree-sidx-idxonly-01.plan | 12 + .../btree-index/btree-sidx-idxonly-01_ps.plan | 33 + .../btree-index/btree-sidx-idxonly-10.plan | 15 + ...nverted-btree-search-return-optional-field.plan | 26 + .../hash-join-with-redundant-variable.1.plan | 74 + .../index-through-object.1.plan | 11 + .../index-through-object.2.plan | 11 + .../index-through-object.3.plan | 11 + .../index-through-object.4.plan | 11 + .../index-through-object.8.plan | 11 + .../index-through-object.9.plan | 41 + .../inverted-index-basic/ngram-contains.plan | 9 + .../inverted-index-basic/ngram-contains_ps.plan | 28 + .../meta/indexes_on_dataset_with_meta_08.plan | 10 + .../multipart-dataverse/index/index-01.plan | 8 + .../multipart-dataverse/index/index-02.plan | 8 + .../inverted-index-basic/ngram-contains_ps.plan | 28 + .../btree-index/non-enforced-composite-key/01.plan | 8 + .../btree-index/non-enforced-composite-key/02.plan | 8 + .../btree-index/non-enforced-composite-key/03.plan | 15 + .../btree-index/non-enforced-composite-key/04.plan | 17 + .../btree-index/non-enforced-composite-key/05.plan | 17 + .../btree-index/non-enforced-composite-key/06.plan | 17 + .../btree-index/non-enforced-composite-key/07.plan | 17 + .../btree-index/non-enforced-composite-key/08.plan | 17 + .../btree-index/non-enforced-composite-key/09.plan | 17 + .../btree-index/non-enforced-composite-key/10.plan | 17 + .../btree-index/non-enforced-composite-key/11.plan | 18 + .../btree-index/non-enforced-composite-key/12.plan | 17 + .../inverted-index-basic/ngram-contains_ps.plan | 28 + .../inverted-index-basic/ngram-contains_ps.plan | 28 + .../btree-composite-key-non-enforced-03.plan | 10 + .../btree-composite-key-non-enforced-04.plan | 10 + .../btree-index-non-enforced-04.plan | 15 + .../btree-index-non-enforced-04_ps.plan | 34 + .../btree-index-non-enforced-05.plan | 15 + .../btree-index-non-enforced-05_ps.plan | 34 + .../btree-index-non-enforced-06.plan | 15 + .../btree-index-non-enforced-06_ps.plan | 34 + .../btree-index-non-enforced-07.plan | 15 + .../btree-index-non-enforced-07_ps.plan | 34 + .../btree-index-non-enforced-08.plan | 15 + .../btree-index-non-enforced-08_ps.plan | 34 + .../btree-index-non-enforced-09.plan | 15 + .../btree-index-non-enforced-09_ps.plan | 34 + .../btree-index-non-enforced-10.plan | 15 + .../btree-index-non-enforced-105.plan | 15 + .../btree-index-non-enforced-105_ps.plan | 34 + .../btree-index-non-enforced-10_ps.plan | 34 + .../btree-index-non-enforced-11.plan | 16 + .../btree-index-non-enforced-11_ps.plan | 36 + .../orders-index-search-conjunctive-open_01.plan | 25 + ...orders-index-search-conjunctive-open_01_ps.plan | 46 + .../orders-index-search-conjunctive-open_02.plan | 25 + ...orders-index-search-conjunctive-open_02_ps.plan | 46 + .../orders-index-search-conjunctive_01.plan | 25 + .../orders-index-search-conjunctive_01_ps.plan | 46 + .../orders-index-search-conjunctive_02.plan | 25 + .../orders-index-search-conjunctive_02_ps.plan | 46 + .../rtree-sidx-idxonly-01.plan | 8 + .../rtree-sidx-idxonly-02.plan | 8 + .../rtree-sidx-idxonly-03.plan | 8 + .../rtree-sidx-idxonly-04.plan | 8 + .../rtree-sidx-idxonly-05.plan | 8 + .../rtree-sidx-idxonly-06.plan | 8 + .../rtree-sidx-idxonly-07.plan | 8 + .../rtree-sidx-idxonly-01-disable-idxonly.plan | 13 + .../rtree-index/rtree-sidx-idxonly-01.plan | 13 + .../skip-index/skip-secondary-btree-index-2.plan | 10 + .../statement-params/statement-params-01.plan | 10 + .../statement-params/statement-params-02.plan | 10 + .../statement-params/statement-params-03.plan | 10 + .../statement-params-index-01.plan | 12 + .../statement-params-index-02.plan | 12 + .../json/json/external_dataset.000.ddl.sqlpp | 23 + .../json/json/external_dataset.008.query.sqlpp} | 7 +- .../json/json/external_dataset.009.query.sqlpp} | 7 +- .../json/json/external_dataset.010.query.sqlpp} | 7 +- .../json/json/external_dataset.011.query.sqlpp} | 9 +- .../json/json/external_dataset.012.query.sqlpp} | 9 +- .../json/json/external_dataset.013.query.sqlpp} | 12 +- .../json/json/external_dataset.014.query.sqlpp} | 12 +- .../hash-join-with-redundant-variable.01.ddl.sqlpp | 134 ++ ...sh-join-with-redundant-variable.02.update.sqlpp | 36 + ...sh-join-with-redundant-variable.03.query.sqlpp} | 20 +- ...sh-join-with-redundant-variable.04.query.sqlpp} | 20 +- ...sh-join-with-redundant-variable.05.query.sqlpp} | 23 +- ...sh-join-with-redundant-variable.06.query.sqlpp} | 23 +- ...sh-join-with-redundant-variable.07.query.sqlpp} | 23 +- ...sh-join-with-redundant-variable.08.query.sqlpp} | 23 +- ...sh-join-with-redundant-variable.09.query.sqlpp} | 23 +- ...sh-join-with-redundant-variable.10.query.sqlpp} | 23 +- ...sh-join-with-redundant-variable.11.query.sqlpp} | 23 +- ...sh-join-with-redundant-variable.12.query.sqlpp} | 23 +- ...sh-join-with-redundant-variable.13.query.sqlpp} | 23 +- ...sh-join-with-redundant-variable.14.query.sqlpp} | 23 +- ...ash-join-with-redundant-variable.15.query.sqlpp | 40 + ...ash-join-with-redundant-variable.16.query.sqlpp | 40 + ...sh-limit-to-primary-lookup-select.3.query.sqlpp | 2 +- ...sh-limit-to-primary-lookup-select.5.query.sqlpp | 2 +- .../push-limit-to-primary-lookup.3.query.sqlpp | 2 +- .../push-limit-to-primary-lookup.5.query.sqlpp | 2 +- .../secondary-index.3.metrics.sqlpp | 4 + .../misc/dump_index/dump_index.1.ddl.sqlpp | 6 +- ...ex.2.update.sqlpp => dump_index.10.query.sqlpp} | 3 +- .../misc/dump_index/dump_index.2.update.sqlpp | 4 +- .../negative/negative.001.ddl.sqlpp} | 14 +- .../negative/negative.002.query.sqlpp} | 4 +- .../negative/negative.003.query.sqlpp} | 4 +- .../negative/negative.004.query.sqlpp} | 4 +- .../negative/negative.005.query.sqlpp} | 4 +- .../negative/negative.999.ddl.sqlpp} | 4 +- .../q01/q01.001.ddl.sqlpp} | 29 +- .../query_index/q01/q01.002.update.sqlpp | 65 + .../q01/q01.003.query.sqlpp} | 5 +- .../q01/q01.004.query.sqlpp} | 5 +- .../q01/q01.005.query.sqlpp} | 5 +- .../q01/q01.006.query.sqlpp} | 5 +- .../q01/q01.007.query.sqlpp} | 5 +- .../q01/q01.008.query.sqlpp} | 5 +- .../q01/q01.009.query.sqlpp} | 5 +- .../q01/q01.010.query.sqlpp} | 5 +- .../q01/q01.011.query.sqlpp} | 5 +- .../q01/q01.012.query.sqlpp} | 5 +- .../q01/q01.013.query.sqlpp} | 5 +- .../q01/q01.014.query.sqlpp} | 5 +- .../q01/q01.015.query.sqlpp} | 4 +- .../q01/q01.016.query.sqlpp} | 3 +- .../q01/q01.017.query.sqlpp} | 4 +- .../q01/q01.018.query.sqlpp} | 4 +- .../q01/q01.019.query.sqlpp} | 4 +- .../q01/q01.020.query.sqlpp} | 4 +- .../q01/q01.021.query.sqlpp} | 4 +- .../q01/q01.022.query.sqlpp} | 6 +- .../q01/q01.023.query.sqlpp} | 6 +- .../q01/q01.024.query.sqlpp} | 4 +- .../q01/q01.025.query.sqlpp} | 4 +- .../q01/q01.026.query.sqlpp} | 10 +- .../q01/q01.027.query.sqlpp} | 4 +- .../q01/q01.028.query.sqlpp} | 4 +- .../q01/q01.029.query.sqlpp} | 4 +- .../q01/q01.030.query.sqlpp} | 4 +- .../q01/q01.031.query.sqlpp} | 10 +- .../q01/q01.999.ddl.sqlpp} | 4 +- .../query-ASTERIXDB-3116.1.query.sqlpp} | 17 +- .../query-ASTERIXDB-3116.2.query.sqlpp} | 18 +- .../count_dataset/count_dataset.1.plan | 22 +- .../aggregate/count_dataset/count_dataset.1.plan | 22 +- .../api/cluster_state_1/cluster_state_1.1.regexadm | 1 - .../cluster_state_1_full.1.regexadm | 1 - .../cluster_state_1_less.1.regexadm | 1 - .../api/compileonly/compileonly.2.regexjson | 1 + .../array_fun/array_remove/array_remove.5.plan | 2 - .../explain_field_access.1.plan | 28 +- .../explain_field_access_closed.1.plan | 28 +- .../explain_object_constructor-01.1.plan | 2 - .../explain_object_constructor-02.1.plan | 2 - .../explain_object_constructor-03.1.plan | 2 - .../explain/explain_simple/explain_simple.1.plan | 2 - .../common/json/json/external_dataset.008.adm | 25 + .../common/json/json/external_dataset.009.adm | 25 + .../common/json/json/external_dataset.010.adm | 1 + .../common/json/json/external_dataset.011.plan | 48 + .../common/json/json/external_dataset.012.adm | 1 + .../common/json/json/external_dataset.013.plan | 62 + .../common/json/json/external_dataset.014.adm | 1 + .../array-access-pushdown.03.plan | 20 +- .../array-access-pushdown.05.plan | 20 +- .../array-access-pushdown.07.plan | 30 +- .../array-access-pushdown.09.plan | 30 +- .../array-access-pushdown.11.plan | 22 +- .../array-access-pushdown.13.plan | 22 +- .../array-access-pushdown.15.plan | 30 +- .../array-access-pushdown.17.plan | 30 +- .../array-access-pushdown.19.plan | 20 +- .../array-access-pushdown.21.plan | 20 +- .../field-access-pushdown.03.plan | 10 +- .../field-access-pushdown.05.plan | 10 +- .../field-access-pushdown.07.plan | 30 +- .../field-access-pushdown.09.plan | 30 +- .../field-access-pushdown.11.plan | 34 +- .../field-access-pushdown.13.plan | 34 +- .../field-access-pushdown.15.plan | 24 +- .../field-access-pushdown.17.plan | 24 +- .../field-access-pushdown.19.plan | 8 +- .../field-access-pushdown.21.plan | 8 +- .../field-access-pushdown.22.plan | 24 +- .../heterogeneous-access-pushdown.03.plan | 22 +- .../heterogeneous-access-pushdown.05.plan | 22 +- .../parquet/object-concat/object-concat.3.plan | 16 +- .../parquet/object-concat/object-concat.5.plan | 20 +- .../parquet/pushdown-plans/pushdown-plans.02.plan | 30 +- .../parquet/pushdown-plans/pushdown-plans.03.plan | 6 +- .../parquet/pushdown-plans/pushdown-plans.04.plan | 38 +- .../parquet/pushdown-plans/pushdown-plans.05.plan | 40 +- .../parquet/pushdown-plans/pushdown-plans.06.plan | 24 +- .../parquet/pushdown-plans/pushdown-plans.07.plan | 82 +- .../common/query-with-limit-plan/result.001.plan | 8 +- .../deterministic/deterministic.4.plan | 2 - .../hash-join-with-redundant-variable.03.adm | 58 + .../hash-join-with-redundant-variable.04.plan | 64 + .../hash-join-with-redundant-variable.05.adm | 3 + .../hash-join-with-redundant-variable.06.plan | 50 + .../hash-join-with-redundant-variable.07.adm | 1500 ++++++++++++++++++++ .../hash-join-with-redundant-variable.08.plan | 50 + .../hash-join-with-redundant-variable.09.adm | 58 + .../hash-join-with-redundant-variable.10.plan | 64 + .../hash-join-with-redundant-variable.11.adm | 58 + .../hash-join-with-redundant-variable.12.plan | 64 + .../hash-join-with-redundant-variable.13.adm | 58 + .../hash-join-with-redundant-variable.14.plan | 66 + .../hash-join-with-redundant-variable.15.adm | 58 + .../hash-join-with-redundant-variable.16.plan | 66 + .../offset_without_limit.6.plan | 12 +- .../push-limit-to-external-scan-select.2.plan | 20 +- .../push-limit-to-external-scan.2.plan | 8 +- .../push-limit-to-primary-lookup-select.3.plan | 16 +- .../push-limit-to-primary-lookup-select.5.plan | 26 +- .../push-limit-to-primary-lookup.3.plan | 16 +- .../push-limit-to-primary-lookup.5.plan | 16 +- .../push-limit-to-primary-scan-select.11.plan | 20 +- .../push-limit-to-primary-scan-select.3.plan | 20 +- .../push-limit-to-primary-scan-select.5.plan | 40 +- .../push-limit-to-primary-scan-select.6.plan | 28 +- .../push-limit-to-primary-scan-select.8.plan | 28 +- .../push-limit-to-primary-scan.3.plan | 6 +- .../push-limit-to-primary-scan.5.plan | 6 +- .../push-limit-to-primary-scan.7.plan | 20 +- .../push-limit-to-primary-scan.8.plan | 20 +- .../enforcing_item_type/enforcing_item_type.1.plan | 8 +- .../misc/constant_folding/constant_folding.1.plan | 2 - .../misc/constant_folding/constant_folding.3.plan | 6 +- .../misc/constant_folding/constant_folding.5.plan | 8 +- .../misc/constant_folding/constant_folding.6.plan | 8 +- .../misc/constant_folding/constant_folding.7.plan | 6 +- .../results/misc/dump_index/dump_index.10.adm | 1 + .../load-record-fields/load-record-fields.4.plan | 14 +- .../load-record-fields/load-record-fields.6.plan | 14 +- .../runtimets/results/query_index/q01/q01.003.adm | 14 + .../runtimets/results/query_index/q01/q01.004.adm | 14 + .../runtimets/results/query_index/q01/q01.005.adm | 14 + .../runtimets/results/query_index/q01/q01.006.adm | 14 + .../runtimets/results/query_index/q01/q01.007.adm | 14 + .../runtimets/results/query_index/q01/q01.008.adm | 14 + .../runtimets/results/query_index/q01/q01.009.adm | 14 + .../runtimets/results/query_index/q01/q01.010.adm | 14 + .../runtimets/results/query_index/q01/q01.011.adm | 8 + .../runtimets/results/query_index/q01/q01.012.adm | 8 + .../runtimets/results/query_index/q01/q01.013.adm | 8 + .../runtimets/results/query_index/q01/q01.014.adm | 8 + .../runtimets/results/query_index/q01/q01.015.adm | 1 + .../runtimets/results/query_index/q01/q01.016.adm | 1 + .../runtimets/results/query_index/q01/q01.017.adm | 4 + .../runtimets/results/query_index/q01/q01.018.adm | 4 + .../runtimets/results/query_index/q01/q01.019.adm | 4 + .../runtimets/results/query_index/q01/q01.020.adm | 4 + .../runtimets/results/query_index/q01/q01.021.adm | 12 + .../runtimets/results/query_index/q01/q01.022.adm | 12 + .../runtimets/results/query_index/q01/q01.023.adm | 12 + .../runtimets/results/query_index/q01/q01.024.plan | 20 + .../runtimets/results/query_index/q01/q01.025.plan | 20 + .../runtimets/results/query_index/q01/q01.026.plan | 38 + .../runtimets/results/query_index/q01/q01.027.plan | 22 + .../runtimets/results/query_index/q01/q01.028.plan | 38 + .../runtimets/results/query_index/q01/q01.029.plan | 38 + .../runtimets/results/query_index/q01/q01.030.plan | 38 + .../runtimets/results/query_index/q01/q01.031.plan | 38 + .../single_dataset_with_index.13.plan | 34 +- .../single_dataset_with_index.8.plan | 34 +- .../query-ASTERIXDB-3116.1.adm | 1 + .../query-ASTERIXDB-3116.2.adm | 1 + .../results/union/union_opt_1/union_opt_1.11.plan | 48 +- .../results/union/union_opt_1/union_opt_1.9.plan | 24 +- .../union/union_type_cast/union_type_cast.4.plan | 2 - .../view/view-pushdown/view-pushdown.04.plan | 34 +- .../view/view-pushdown/view-pushdown.06.plan | 20 +- .../view/view-pushdown/view-pushdown.08.plan | 26 +- .../view/view-pushdown/view-pushdown.10.plan | 26 +- .../view/view-pushdown/view-pushdown.12.plan | 26 +- .../view/view-pushdown/view-pushdown.14.plan | 26 +- .../test/resources/runtimets/testsuite_sqlpp.xml | 26 + .../annotations/ExternalSubpathAnnotation.java} | 22 +- .../asterix/common/config/CompilerProperties.java | 2 +- .../asterix/common/exceptions/ErrorCode.java | 1 + .../src/main/resources/asx_errormsg/en.properties | 1 + .../external/util/ExternalDataConstants.java | 1 + .../asterix/external/util/ExternalDataUtils.java | 27 +- .../external/util/google/gcs/GCSConstants.java | 1 + .../asterix/external/util/google/gcs/GCSUtils.java | 31 +- .../asterix/lang/common/util/FunctionUtil.java | 69 + .../asterix/lang/sqlpp/parser/SqlppHint.java | 1 + .../visitor/Sql92AggregateFunctionVisitor.java | 3 +- .../visitor/VariableCheckAndRewriteVisitor.java | 1 + .../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 23 +- .../lang/expression/CommutativeEqualsTest.java | 87 ++ .../metadata/declared/DatasetDataSource.java | 20 +- .../metadata/declared/FunctionDataSource.java | 13 +- .../asterix/metadata/utils/KeyFieldTypeUtil.java | 8 +- asterixdb/pom.xml | 90 +- .../core/algebra/metadata/IDataSource.java | 14 + .../visitors/IsomorphismOperatorVisitor.java | 7 + .../AbstractLogicalOperatorPrettyPrintVisitor.java | 1 - .../LogicalOperatorPrettyPrintVisitor.java | 13 +- .../LogicalOperatorPrettyPrintVisitorJson.java | 21 - .../IPartitioningRequirementsCoordinator.java | 26 +- .../algebra/util/OperatorManipulationUtil.java | 22 + hyracks-fullstack/hyracks/hyracks-hdfs/pom.xml | 38 +- hyracks-fullstack/pom.xml | 34 +- 456 files changed, 9257 insertions(+), 1445 deletions(-) diff --cc asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/AbstractOperatorFromSubplanRewrite.java index 587132f562,530596d0b5..33281271c5 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/AbstractOperatorFromSubplanRewrite.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/AbstractOperatorFromSubplanRewrite.java @@@ -509,7 -537,9 +537,7 @@@ abstract public class AbstractOperatorF if (splitIntoConjuncts(conjunct.getValue(), innerExprConjuncts)) { conjuncts.addAll(innerExprConjuncts); } else { - conjuncts.add(new MutableObject<>(conjunct.getValue().cloneExpression())); - + conjuncts.add(new MutableObject<>(conjunct.getValue())); - } } return true; diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java index 597e7cc950,be21673b33..41be44b153 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java @@@ -320,8 -338,7 +325,8 @@@ public class APIFramework if (conf.is(SessionConfig.OOB_OPTIMIZED_LOGICAL_PLAN) || isExplainOnly) { if (isQuery || isLoad) { - generateOptimizedLogicalPlan(plan, output.config().getPlanFormat(), cboMode); + generateOptimizedLogicalPlan(plan, spec.getLogical2PhysicalMap(), output.config().getPlanFormat(), - isExplainOnly); ++ cboMode); } } diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java index 0000000000,cf2b891357..52cd17c878 mode 000000,100644..100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java @@@ -1,0 -1,129 +1,131 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.asterix.app.function; + + import java.util.ArrayList; + import java.util.List; + + import org.apache.asterix.common.cluster.IClusterStateManager; + import org.apache.asterix.metadata.api.IDatasourceFunction; + import org.apache.asterix.metadata.declared.DataSourceId; + import org.apache.asterix.metadata.declared.FunctionDataSource; + import org.apache.asterix.metadata.declared.MetadataProvider; + import org.apache.asterix.metadata.entities.Dataset; + import org.apache.asterix.om.types.ARecordType; + import org.apache.asterix.om.types.IAType; + import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; + import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; + import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; + import org.apache.hyracks.algebricks.common.utils.Pair; + import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; + import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; + import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource; + import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider; + import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionInfo; + import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; + import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator; + import org.apache.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty; + import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain; + import org.apache.hyracks.algebricks.core.algebra.properties.LocalOrderProperty; + import org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn; + import org.apache.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty; + import org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector; + import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext; + import org.apache.hyracks.api.dataflow.IOperatorDescriptor; + import org.apache.hyracks.api.job.JobSpecification; + import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory; ++import org.apache.hyracks.storage.am.common.impls.DefaultTupleProjectorFactory; + + public class QueryIndexDatasource extends FunctionDataSource { + + private final Dataset ds; + private final String indexName; + private final AlgebricksAbsolutePartitionConstraint storageLocations; + private final int numSecKeys; + + public QueryIndexDatasource(Dataset ds, String indexName, INodeDomain domain, + AlgebricksAbsolutePartitionConstraint storageLocations, ARecordType recType, int numSecKeys) + throws AlgebricksException { + super(createQueryIndexDataSourceId(ds, indexName), QueryIndexRewriter.QUERY_INDEX, domain, recType); + this.ds = ds; + this.indexName = indexName; + this.storageLocations = storageLocations; + this.numSecKeys = numSecKeys; + } + + @Override + protected void initSchemaType(IAType iType) { + ARecordType type = (ARecordType) iType; + IAType[] fieldTypes = type.getFieldTypes(); + schemaTypes = new IAType[fieldTypes.length]; + for (int i = 0; i < schemaTypes.length; i++) { + schemaTypes[i] = fieldTypes[i]; + } + } + + @Override + protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) { + return storageLocations; + } + + @Override + public boolean isScanAccessPathALeaf() { + // the index scan op is not a leaf op. the ETS op will start the scan of the index. we need the ETS op below + // the index scan to be still generated + return false; + } + + @Override + protected IDatasourceFunction createFunction(MetadataProvider metadataProvider, + AlgebricksAbsolutePartitionConstraint locations) { + throw new UnsupportedOperationException("query-index() does not use record reader adapter"); + } + + @Override + public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildDatasourceScanRuntime( + MetadataProvider metadataProvider, IDataSource<DataSourceId> dataSource, + List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed, + List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, + ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema, + IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig, + IProjectionInfo<?> projectionInfo) throws AlgebricksException { + return metadataProvider.buildBtreeRuntime(jobSpec, opSchema, typeEnv, context, true, false, null, ds, indexName, - null, null, true, true, false, null, null, null, tupleFilterFactory, outputLimit, false, false); ++ null, null, true, true, false, null, null, null, tupleFilterFactory, outputLimit, false, false, ++ DefaultTupleProjectorFactory.INSTANCE); + } + + @Override + public IDataSourcePropertiesProvider getPropertiesProvider() { + return scanVariables -> { + List<ILocalStructuralProperty> propsLocal = new ArrayList<>(1); + //TODO(ali): consider primary keys? + List<OrderColumn> secKeys = new ArrayList<>(numSecKeys); + for (int i = 0; i < numSecKeys; i++) { + secKeys.add(new OrderColumn(scanVariables.get(i), OrderOperator.IOrder.OrderKind.ASC)); + } + propsLocal.add(new LocalOrderProperty(secKeys)); + return new StructuralPropertiesVector(new RandomPartitioningProperty(domain), propsLocal); + }; + } + + private static DataSourceId createQueryIndexDataSourceId(Dataset dataset, String indexName) { + return new DataSourceId(dataset.getDataverseName(), dataset.getDatasetName(), + new String[] { indexName, QueryIndexRewriter.QUERY_INDEX.getName() }); + } + } diff --cc asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.1.plan index 0000000000,0000000000..f0f6a0394c new file mode 100644 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.1.plan @@@ -1,0 -1,0 +1,11 @@@ ++-- DISTRIBUTE_RESULT |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- STREAM_PROJECT |PARTITIONED| ++ -- ASSIGN |PARTITIONED| ++ -- STREAM_SELECT |PARTITIONED| ++ -- ASSIGN |PARTITIONED| ++ -- STREAM_PROJECT |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --cc asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.2.plan index 0000000000,0000000000..f0f6a0394c new file mode 100644 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.2.plan @@@ -1,0 -1,0 +1,11 @@@ ++-- DISTRIBUTE_RESULT |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- STREAM_PROJECT |PARTITIONED| ++ -- ASSIGN |PARTITIONED| ++ -- STREAM_SELECT |PARTITIONED| ++ -- ASSIGN |PARTITIONED| ++ -- STREAM_PROJECT |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --cc asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.3.plan index 0000000000,0000000000..f0f6a0394c new file mode 100644 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.3.plan @@@ -1,0 -1,0 +1,11 @@@ ++-- DISTRIBUTE_RESULT |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- STREAM_PROJECT |PARTITIONED| ++ -- ASSIGN |PARTITIONED| ++ -- STREAM_SELECT |PARTITIONED| ++ -- ASSIGN |PARTITIONED| ++ -- STREAM_PROJECT |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --cc asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.4.plan index 0000000000,0000000000..f0f6a0394c new file mode 100644 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.4.plan @@@ -1,0 -1,0 +1,11 @@@ ++-- DISTRIBUTE_RESULT |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- STREAM_PROJECT |PARTITIONED| ++ -- ASSIGN |PARTITIONED| ++ -- STREAM_SELECT |PARTITIONED| ++ -- ASSIGN |PARTITIONED| ++ -- STREAM_PROJECT |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --cc asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.8.plan index 0000000000,0000000000..f0f6a0394c new file mode 100644 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.8.plan @@@ -1,0 -1,0 +1,11 @@@ ++-- DISTRIBUTE_RESULT |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- STREAM_PROJECT |PARTITIONED| ++ -- ASSIGN |PARTITIONED| ++ -- STREAM_SELECT |PARTITIONED| ++ -- ASSIGN |PARTITIONED| ++ -- STREAM_PROJECT |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --cc asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.9.plan index 0000000000,0000000000..75a50b4f76 new file mode 100644 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/index-through-object/index-through-object.9.plan @@@ -1,0 -1,0 +1,41 @@@ ++-- DISTRIBUTE_RESULT |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- STREAM_PROJECT |PARTITIONED| ++ -- ASSIGN |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- NESTED_LOOP |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- STREAM_PROJECT |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- HYBRID_HASH_JOIN [$$88][$$89] |PARTITIONED| ++ -- HASH_PARTITION_EXCHANGE [$$88] |PARTITIONED| ++ -- ASSIGN |PARTITIONED| ++ -- STREAM_SELECT |PARTITIONED| ++ -- ASSIGN |PARTITIONED| ++ -- STREAM_PROJECT |PARTITIONED| ++ -- ASSIGN |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- REPLICATE |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- STREAM_PROJECT |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- EMPTY_TUPLE_SOURCE |PARTITIONED| ++ -- HASH_PARTITION_EXCHANGE [$$89] |PARTITIONED| ++ -- STREAM_SELECT |PARTITIONED| ++ -- ASSIGN |PARTITIONED| ++ -- ASSIGN |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- REPLICATE |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- STREAM_PROJECT |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- DATASOURCE_SCAN (Test.Users) |PARTITIONED| ++ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| ++ -- EMPTY_TUPLE_SOURCE |PARTITIONED| ++ -- BROADCAST_EXCHANGE |PARTITIONED| ++ -- STREAM_SELECT |UNPARTITIONED| ++ -- ASSIGN |UNPARTITIONED| ++ -- UNNEST |UNPARTITIONED| ++ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED| diff --cc asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_01.plan index 0000000000,2326c55cad..e4167f0e90 mode 000000,100644..100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_01.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_01.plan @@@ -1,0 -1,25 +1,25 @@@ + -- SINK |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BULKLOAD |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$2(ASC)] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$2] |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (loadable_dv.loadable_ds) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- DISTRIBUTE_RESULT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| - -- SORT_MERGE_EXCHANGE [$$23(ASC) ] |PARTITIONED| ++ -- SORT_MERGE_EXCHANGE [$$24(ASC) ] |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (tpch.Orders) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --cc asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_01_ps.plan index 0000000000,4bdc66bb30..8e1370f459 mode 000000,100644..100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_01_ps.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_01_ps.plan @@@ -1,0 -1,46 +1,46 @@@ + -- SINK |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BULKLOAD |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$2(ASC)] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$2] |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (loadable_dv.loadable_ds) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- DISTRIBUTE_RESULT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STABLE_SORT [$$23(ASC)] |PARTITIONED| - -- RANGE_PARTITION_EXCHANGE [$$23(ASC)] |PARTITIONED| ++ -- STABLE_SORT [$$24(ASC)] |PARTITIONED| ++ -- RANGE_PARTITION_EXCHANGE [$$24(ASC)] |PARTITIONED| + -- FORWARD |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (tpch.Orders) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- BROADCAST_EXCHANGE |PARTITIONED| + -- AGGREGATE |UNPARTITIONED| + -- RANDOM_MERGE_EXCHANGE |PARTITIONED| + -- AGGREGATE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (tpch.Orders) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --cc asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_02.plan index 0000000000,eb1d57c76d..c87d9ce8f9 mode 000000,100644..100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_02.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_02.plan @@@ -1,0 -1,25 +1,25 @@@ + -- SINK |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BULKLOAD |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$2(ASC)] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$2] |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (loadable_dv.loadable_ds) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- DISTRIBUTE_RESULT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| - -- SORT_MERGE_EXCHANGE [$$25(ASC) ] |PARTITIONED| ++ -- SORT_MERGE_EXCHANGE [$$26(ASC) ] |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (tpch.Orders) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --cc asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_02_ps.plan index 0000000000,dce703e784..6db1de83e8 mode 000000,100644..100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_02_ps.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive-open_02_ps.plan @@@ -1,0 -1,46 +1,46 @@@ + -- SINK |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BULKLOAD |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$2(ASC)] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$2] |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (loadable_dv.loadable_ds) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- DISTRIBUTE_RESULT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STABLE_SORT [$$25(ASC)] |PARTITIONED| - -- RANGE_PARTITION_EXCHANGE [$$25(ASC)] |PARTITIONED| ++ -- STABLE_SORT [$$26(ASC)] |PARTITIONED| ++ -- RANGE_PARTITION_EXCHANGE [$$26(ASC)] |PARTITIONED| + -- FORWARD |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (tpch.Orders) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- BROADCAST_EXCHANGE |PARTITIONED| + -- AGGREGATE |UNPARTITIONED| + -- RANDOM_MERGE_EXCHANGE |PARTITIONED| + -- AGGREGATE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (tpch.Orders) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --cc asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_01.plan index 0000000000,2326c55cad..e4167f0e90 mode 000000,100644..100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_01.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_01.plan @@@ -1,0 -1,25 +1,25 @@@ + -- SINK |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BULKLOAD |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$2(ASC)] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$2] |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (loadable_dv.loadable_ds) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- DISTRIBUTE_RESULT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| - -- SORT_MERGE_EXCHANGE [$$23(ASC) ] |PARTITIONED| ++ -- SORT_MERGE_EXCHANGE [$$24(ASC) ] |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (tpch.Orders) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --cc asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_01_ps.plan index 0000000000,4bdc66bb30..8e1370f459 mode 000000,100644..100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_01_ps.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_01_ps.plan @@@ -1,0 -1,46 +1,46 @@@ + -- SINK |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BULKLOAD |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$2(ASC)] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$2] |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (loadable_dv.loadable_ds) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- DISTRIBUTE_RESULT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STABLE_SORT [$$23(ASC)] |PARTITIONED| - -- RANGE_PARTITION_EXCHANGE [$$23(ASC)] |PARTITIONED| ++ -- STABLE_SORT [$$24(ASC)] |PARTITIONED| ++ -- RANGE_PARTITION_EXCHANGE [$$24(ASC)] |PARTITIONED| + -- FORWARD |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (tpch.Orders) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- BROADCAST_EXCHANGE |PARTITIONED| + -- AGGREGATE |UNPARTITIONED| + -- RANDOM_MERGE_EXCHANGE |PARTITIONED| + -- AGGREGATE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (tpch.Orders) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --cc asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_02.plan index 0000000000,eb1d57c76d..c87d9ce8f9 mode 000000,100644..100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_02.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_02.plan @@@ -1,0 -1,25 +1,25 @@@ + -- SINK |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BULKLOAD |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$2(ASC)] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$2] |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (loadable_dv.loadable_ds) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- DISTRIBUTE_RESULT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| - -- SORT_MERGE_EXCHANGE [$$25(ASC) ] |PARTITIONED| ++ -- SORT_MERGE_EXCHANGE [$$26(ASC) ] |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (tpch.Orders) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --cc asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_02_ps.plan index 0000000000,dce703e784..6db1de83e8 mode 000000,100644..100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_02_ps.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_cbo/orders-index-search-conjunctive_02_ps.plan @@@ -1,0 -1,46 +1,46 @@@ + -- SINK |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BULKLOAD |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$2(ASC)] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$2] |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (loadable_dv.loadable_ds) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- DISTRIBUTE_RESULT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STABLE_SORT [$$25(ASC)] |PARTITIONED| - -- RANGE_PARTITION_EXCHANGE [$$25(ASC)] |PARTITIONED| ++ -- STABLE_SORT [$$26(ASC)] |PARTITIONED| ++ -- RANGE_PARTITION_EXCHANGE [$$26(ASC)] |PARTITIONED| + -- FORWARD |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (tpch.Orders) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- BROADCAST_EXCHANGE |PARTITIONED| + -- AGGREGATE |UNPARTITIONED| + -- RANDOM_MERGE_EXCHANGE |PARTITIONED| + -- AGGREGATE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN (tpch.Orders) |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --cc asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm index d90e1f89e6,dd6ac17d84..389b120a23 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm @@@ -10,9 -10,7 +10,8 @@@ "active\.suspend\.timeout" : 3600, "azure.request.timeout" : 120, "compiler\.arrayindex" : true, + "compiler.batch.lookup" : false, "compiler.cbo" : false, - "compiler.cbotest" : true, "compiler\.external\.field\.pushdown" : true, "compiler.forcejoinorder" : false, "compiler\.framesize" : 32768, diff --cc asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm index efa091cd53,9103663bfd..e68df5f736 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm @@@ -10,9 -10,7 +10,8 @@@ "active\.suspend\.timeout" : 3600, "azure.request.timeout" : 120, "compiler\.arrayindex" : true, + "compiler.batch.lookup" : false, "compiler.cbo" : false, - "compiler.cbotest" : true, "compiler\.external\.field\.pushdown" : true, "compiler.forcejoinorder" : false, "compiler\.framesize" : 32768, diff --cc asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm index 732f05cd04,7ae9886fb5..a6fa211468 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm @@@ -10,9 -10,7 +10,8 @@@ "active\.suspend\.timeout" : 3600, "azure.request.timeout" : 120, "compiler\.arrayindex" : true, + "compiler.batch.lookup" : false, "compiler.cbo" : false, - "compiler.cbotest" : true, "compiler\.external\.field\.pushdown" : true, "compiler.forcejoinorder" : false, "compiler\.framesize" : 32768, diff --cc asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.13.plan index 64e9722d8c,6faed2fde1..c69e8a0d87 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.13.plan +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.13.plan @@@ -1,34 -1,32 +1,32 @@@ - cardinality: 1000000.0 - cost: 1000000.0 - distribute result [$$l] [cardinality: 1000000.0, op-cost: 0.0, total-cost: 1000000.0] + distribute result [$$l] -- DISTRIBUTE_RESULT |PARTITIONED| - exchange [cardinality: 1000000.0, op-cost: 0.0, total-cost: 1000000.0] + exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - project ([$$l]) [cardinality: 1000000.0, op-cost: 0.0, total-cost: 1000000.0] + project ([$$l]) -- STREAM_PROJECT |PARTITIONED| - exchange [cardinality: 1000000.0, op-cost: 0.0, total-cost: 1000000.0] + exchange - -- SORT_MERGE_EXCHANGE [$$16(ASC), $$17(ASC) ] |PARTITIONED| + -- SORT_MERGE_EXCHANGE [$$17(ASC), $$18(ASC) ] |PARTITIONED| - select (eq($$l.getField(10), "1994-01-20")) [cardinality: 1000000.0, op-cost: 0.0, total-cost: 1000000.0] + select (eq($$l.getField(10), "1994-01-20")) -- STREAM_SELECT |PARTITIONED| - exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - unnest-map [$$17, $$18, $$l] <- index-search("LineItem", 0, "tpch", "LineItem", false, false, 2, $$25, $$26, 2, $$25, $$26, true, true, true) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] - unnest-map [$$16, $$17, $$l] <- index-search("LineItem", 0, "tpch", "LineItem", false, false, 2, $$24, $$25, 2, $$24, $$25, true, true, true) ++ unnest-map [$$17, $$18, $$l] <- index-search("LineItem", 0, "tpch", "LineItem", false, false, 2, $$25, $$26, 2, $$25, $$26, true, true, true) -- BTREE_SEARCH |PARTITIONED| - exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - order (ASC, $$25) (ASC, $$26) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] - order (ASC, $$24) (ASC, $$25) - -- STABLE_SORT [$$24(ASC), $$25(ASC)] |PARTITIONED| ++ order (ASC, $$25) (ASC, $$26) + -- STABLE_SORT [$$25(ASC), $$26(ASC)] |PARTITIONED| - exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - project ([$$25, $$26]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] - project ([$$24, $$25]) ++ project ([$$25, $$26]) -- STREAM_PROJECT |PARTITIONED| - exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - unnest-map [$$24, $$25, $$26] <- index-search("lineitem_shipdateIx", 0, "tpch", "LineItem", false, false, 1, $$22, 1, $$23, true, true, true) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] - unnest-map [$$23, $$24, $$25] <- index-search("lineitem_shipdateIx", 0, "tpch", "LineItem", false, false, 1, $$21, 1, $$22, true, true, true) ++ unnest-map [$$24, $$25, $$26] <- index-search("lineitem_shipdateIx", 0, "tpch", "LineItem", false, false, 1, $$22, 1, $$23, true, true, true) -- BTREE_SEARCH |PARTITIONED| - exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - assign [$$22, $$23] <- ["1994-01-20", "1994-01-20"] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] - assign [$$21, $$22] <- ["1994-01-20", "1994-01-20"] ++ assign [$$22, $$23] <- ["1994-01-20", "1994-01-20"] -- ASSIGN |PARTITIONED| - empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + empty-tuple-source -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --cc asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.8.plan index 64e9722d8c,6faed2fde1..c69e8a0d87 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.8.plan +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.8.plan @@@ -1,34 -1,32 +1,32 @@@ - cardinality: 1000000.0 - cost: 1000000.0 - distribute result [$$l] [cardinality: 1000000.0, op-cost: 0.0, total-cost: 1000000.0] + distribute result [$$l] -- DISTRIBUTE_RESULT |PARTITIONED| - exchange [cardinality: 1000000.0, op-cost: 0.0, total-cost: 1000000.0] + exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - project ([$$l]) [cardinality: 1000000.0, op-cost: 0.0, total-cost: 1000000.0] + project ([$$l]) -- STREAM_PROJECT |PARTITIONED| - exchange [cardinality: 1000000.0, op-cost: 0.0, total-cost: 1000000.0] + exchange - -- SORT_MERGE_EXCHANGE [$$16(ASC), $$17(ASC) ] |PARTITIONED| + -- SORT_MERGE_EXCHANGE [$$17(ASC), $$18(ASC) ] |PARTITIONED| - select (eq($$l.getField(10), "1994-01-20")) [cardinality: 1000000.0, op-cost: 0.0, total-cost: 1000000.0] + select (eq($$l.getField(10), "1994-01-20")) -- STREAM_SELECT |PARTITIONED| - exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - unnest-map [$$17, $$18, $$l] <- index-search("LineItem", 0, "tpch", "LineItem", false, false, 2, $$25, $$26, 2, $$25, $$26, true, true, true) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] - unnest-map [$$16, $$17, $$l] <- index-search("LineItem", 0, "tpch", "LineItem", false, false, 2, $$24, $$25, 2, $$24, $$25, true, true, true) ++ unnest-map [$$17, $$18, $$l] <- index-search("LineItem", 0, "tpch", "LineItem", false, false, 2, $$25, $$26, 2, $$25, $$26, true, true, true) -- BTREE_SEARCH |PARTITIONED| - exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - order (ASC, $$25) (ASC, $$26) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] - order (ASC, $$24) (ASC, $$25) - -- STABLE_SORT [$$24(ASC), $$25(ASC)] |PARTITIONED| ++ order (ASC, $$25) (ASC, $$26) + -- STABLE_SORT [$$25(ASC), $$26(ASC)] |PARTITIONED| - exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - project ([$$25, $$26]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] - project ([$$24, $$25]) ++ project ([$$25, $$26]) -- STREAM_PROJECT |PARTITIONED| - exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - unnest-map [$$24, $$25, $$26] <- index-search("lineitem_shipdateIx", 0, "tpch", "LineItem", false, false, 1, $$22, 1, $$23, true, true, true) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] - unnest-map [$$23, $$24, $$25] <- index-search("lineitem_shipdateIx", 0, "tpch", "LineItem", false, false, 1, $$21, 1, $$22, true, true, true) ++ unnest-map [$$24, $$25, $$26] <- index-search("lineitem_shipdateIx", 0, "tpch", "LineItem", false, false, 1, $$22, 1, $$23, true, true, true) -- BTREE_SEARCH |PARTITIONED| - exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + exchange -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - assign [$$22, $$23] <- ["1994-01-20", "1994-01-20"] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] - assign [$$21, $$22] <- ["1994-01-20", "1994-01-20"] ++ assign [$$22, $$23] <- ["1994-01-20", "1994-01-20"] -- ASSIGN |PARTITIONED| - empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + empty-tuple-source -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --cc asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_type_cast/union_type_cast.4.plan index 99e8280a4d,0000000000..d0aa765b16 mode 100644,000000..100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_type_cast/union_type_cast.4.plan +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_type_cast/union_type_cast.4.plan @@@ -1,74 -1,0 +1,72 @@@ - cardinality: 0.0 - cost: 0.0 +distribute result [$#1] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] +-- DISTRIBUTE_RESULT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$#1]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- SORT_MERGE_EXCHANGE [$$102(ASC) ] |PARTITIONED| + order (ASC, $$102) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STABLE_SORT [$$102(ASC)] |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + union ($$64, $$140, $#1) ($$103, $$70, $$102) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- UNION_ALL |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + union ($$139, $$141, $$64) ($$68, $$69, $$103) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- UNION_ALL |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$139, $$68]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$139] <- [cast({"id": $$68, "no_in_response_to": object-remove($$s, "in-response-to")})] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + replicate [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- REPLICATE |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan []<-[$$68, $$s] <- TinySocial.FacebookMessages [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$141, $$69]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$141] <- [cast({"id": $$69, "user": $$t})] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan []<-[$$69, $$t] <- TinySocial.FacebookUsers [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$140, $$70]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$140] <- [cast({"id": $$70, "no_author_id": object-remove($$s, "author-id")})] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + project ([$$70, $$s]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |PARTITIONED| + assign [$$70, $$s] <- [$$68, $$s] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + replicate [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- REPLICATE |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + data-scan []<-[$$68, $$s] <- TinySocial.FacebookMessages [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --cc asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java index 5bf584487e,29e04e9904..d73089875c --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java @@@ -18,15 -18,51 +18,10 @@@ */ package org.apache.asterix.external.util; -import static com.google.cloud.storage.Storage.BlobListOption; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.asterix.common.exceptions.ErrorCode.EXTERNAL_SOURCE_ERROR; -import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_PARAM_VALUE_ALLOWED_VALUE; --import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_REQ_PARAM_VAL; --import static org.apache.asterix.common.exceptions.ErrorCode.PARAMETERS_NOT_ALLOWED_AT_SAME_TIME; --import static org.apache.asterix.common.exceptions.ErrorCode.PARAMETERS_REQUIRED; -import static org.apache.asterix.common.exceptions.ErrorCode.PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT; -import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_IF_PARAM_IS_PRESENT; -import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_OR_PARAM_IF_PARAM_IS_PRESENT; -import static org.apache.asterix.common.exceptions.ErrorCode.S3_REGION_NOT_SUPPORTED; -import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME; -import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_METHOD_NOT_IMPLEMENTED; -import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_ACCESS_KEY_ID; -import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_ANONYMOUS_ACCESS; -import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_CREDENTIAL_PROVIDER_KEY; -import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_PATH_STYLE_ACCESS; -import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_S3_CONNECTION_POOL_SIZE; -import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_S3_PROTOCOL; -import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_SECRET_ACCESS_KEY; -import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_SESSION_TOKEN; -import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_TEMP_ACCESS; -import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME; -import static org.apache.asterix.external.util.ExternalDataConstants.Azure.ACCOUNT_KEY_FIELD_NAME; -import static org.apache.asterix.external.util.ExternalDataConstants.Azure.ACCOUNT_NAME_FIELD_NAME; -import static org.apache.asterix.external.util.ExternalDataConstants.Azure.CLIENT_CERTIFICATE_FIELD_NAME; -import static org.apache.asterix.external.util.ExternalDataConstants.Azure.CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME; -import static org.apache.asterix.external.util.ExternalDataConstants.Azure.CLIENT_ID_FIELD_NAME; -import static org.apache.asterix.external.util.ExternalDataConstants.Azure.CLIENT_SECRET_FIELD_NAME; -import static org.apache.asterix.external.util.ExternalDataConstants.Azure.ENDPOINT_FIELD_NAME; -import static org.apache.asterix.external.util.ExternalDataConstants.Azure.HADOOP_AZURE_BLOB_PROTOCOL; -import static org.apache.asterix.external.util.ExternalDataConstants.Azure.HADOOP_AZURE_FS_ACCOUNT_KEY; -import static org.apache.asterix.external.util.ExternalDataConstants.Azure.HADOOP_AZURE_FS_SAS; -import static org.apache.asterix.external.util.ExternalDataConstants.Azure.MANAGED_IDENTITY_ID_FIELD_NAME; -import static org.apache.asterix.external.util.ExternalDataConstants.Azure.RECURSIVE_FIELD_NAME; -import static org.apache.asterix.external.util.ExternalDataConstants.Azure.SHARED_ACCESS_SIGNATURE_FIELD_NAME; -import static org.apache.asterix.external.util.ExternalDataConstants.Azure.TENANT_ID_FIELD_NAME; -import static org.apache.asterix.external.util.ExternalDataConstants.GCS.APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME; -import static org.apache.asterix.external.util.ExternalDataConstants.GCS.JSON_CREDENTIALS_FIELD_NAME; --import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ADAPTER_NAME_GCS; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_DELIMITER; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ESCAPE; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXCLUDE; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXTERNAL_SCAN_BUFFER_SIZE; --import static org.apache.asterix.external.util.ExternalDataConstants.KEY_FORMAT; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_INCLUDE; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_QUOTE; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_END; diff --cc asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java index 6bf2266de7,0000000000..f2dbde7f80 mode 100644,000000..100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java @@@ -1,46 -1,0 +1,47 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.external.util.google.gcs; + +public class GCSConstants { + private GCSConstants() { + throw new AssertionError("do not instantiate"); + } + ++ public static final String APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME = "applicationDefaultCredentials"; + public static final String JSON_CREDENTIALS_FIELD_NAME = "jsonCredentials"; + public static final String ENDPOINT_FIELD_NAME = "endpoint"; + + /* + * Hadoop internal configuration + */ + public static final String HADOOP_GCS_PROTOCOL = "gs"; + + // hadoop credentials + public static final String HADOOP_AUTH_TYPE = "fs.gs.auth.type"; + public static final String HADOOP_AUTH_UNAUTHENTICATED = "UNAUTHENTICATED"; + public static final String HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE = "SERVICE_ACCOUNT_JSON_KEYFILE"; + public static final String HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE_PATH = + "google.cloud.auth.service.account.json.keyfile"; + + // gs hadoop parameters + public static final String HADOOP_SUPPORT_COMPRESSED = "fs.gs.inputstream.support.gzip.encoding.enable"; + public static final String HADOOP_ENDPOINT = "fs.gs.storage.root.url"; + public static final String HADOOP_MAX_REQUESTS_PER_BATCH = "fs.gs.max.requests.per.batch"; + public static final String HADOOP_BATCH_THREADS = "fs.gs.batch.threads"; +} diff --cc asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java index 93dc272067,0000000000..3efb041dae mode 100644,000000..100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java @@@ -1,217 -1,0 +1,242 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.external.util.google.gcs; + +import static org.apache.asterix.common.exceptions.ErrorCode.EXTERNAL_SOURCE_ERROR; ++import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_PARAM_VALUE_ALLOWED_VALUE; ++import static org.apache.asterix.common.exceptions.ErrorCode.PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT; +import static org.apache.asterix.external.util.ExternalDataUtils.getPrefix; +import static org.apache.asterix.external.util.ExternalDataUtils.validateIncludeExclude; ++import static org.apache.asterix.external.util.google.gcs.GCSConstants.APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME; +import static org.apache.asterix.external.util.google.gcs.GCSConstants.ENDPOINT_FIELD_NAME; +import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE; +import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE_PATH; +import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_AUTH_TYPE; +import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_AUTH_UNAUTHENTICATED; +import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_ENDPOINT; +import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_GCS_PROTOCOL; +import static org.apache.asterix.external.util.google.gcs.GCSConstants.JSON_CREDENTIALS_FIELD_NAME; +import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.BiPredicate; +import java.util.regex.Matcher; + +import org.apache.asterix.common.exceptions.CompilationException; +import org.apache.asterix.common.exceptions.ErrorCode; +import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory.IncludeExcludeMatcher; +import org.apache.asterix.external.util.ExternalDataConstants; +import org.apache.asterix.external.util.ExternalDataUtils; +import org.apache.asterix.external.util.HDFSUtils; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hyracks.api.exceptions.IWarningCollector; +import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.hyracks.api.exceptions.Warning; + +import com.google.api.gax.paging.Page; - import com.google.auth.oauth2.ServiceAccountCredentials; ++import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.BaseServiceException; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; + +public class GCSUtils { + private GCSUtils() { + throw new AssertionError("do not instantiate"); + + } + + /** + * Builds the client using the provided configuration + * + * @param configuration properties + * @return clientasterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java + * @throws CompilationException CompilationException + */ + public static Storage buildClient(Map<String, String> configuration) throws CompilationException { ++ String applicationDefaultCredentials = configuration.get(APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME); + String jsonCredentials = configuration.get(JSON_CREDENTIALS_FIELD_NAME); + String endpoint = configuration.get(ENDPOINT_FIELD_NAME); + + StorageOptions.Builder builder = StorageOptions.newBuilder(); + - // Use credentials if available ++ // default credentials provider ++ if (applicationDefaultCredentials != null) { ++ // only "true" value is allowed ++ if (!applicationDefaultCredentials.equalsIgnoreCase("true")) { ++ throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE, ++ APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME, "true"); ++ } ++ ++ // no other authentication parameters are allowed ++ if (jsonCredentials != null) { ++ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, JSON_CREDENTIALS_FIELD_NAME, ++ APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME); ++ } ++ ++ try { ++ builder.setCredentials(GoogleCredentials.getApplicationDefault()); ++ } catch (IOException ex) { ++ throw CompilationException.create(EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex)); ++ } ++ } ++ ++ // json credentials + if (jsonCredentials != null) { + try (InputStream credentialsStream = new ByteArrayInputStream(jsonCredentials.getBytes())) { - builder.setCredentials(ServiceAccountCredentials.fromStream(credentialsStream)); ++ builder.setCredentials(GoogleCredentials.fromStream(credentialsStream)); + } catch (IOException ex) { + throw new CompilationException(EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex)); + } + } + + if (endpoint != null) { + builder.setHost(endpoint); + } + + return builder.build().getService(); + } + + /** + * Validate external dataset properties + * + * @param configuration properties + * @throws CompilationException Compilation exception + */ + public static void validateProperties(Map<String, String> configuration, SourceLocation srcLoc, + IWarningCollector collector) throws CompilationException { + + // check if the format property is present + if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) { + throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT); + } + + validateIncludeExclude(configuration); + String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME); + + try { + Storage.BlobListOption limitOption = Storage.BlobListOption.pageSize(1); + Storage.BlobListOption prefixOption = Storage.BlobListOption.prefix(getPrefix(configuration)); + Storage storage = buildClient(configuration); + Page<Blob> items = storage.list(container, limitOption, prefixOption); + + if (!items.iterateAll().iterator().hasNext() && collector.shouldWarn()) { + Warning warning = Warning.of(srcLoc, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES); + collector.warn(warning); + } + } catch (CompilationException ex) { + throw ex; + } catch (Exception ex) { + throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex)); + } + } + + public static List<Blob> listItems(Map<String, String> configuration, IncludeExcludeMatcher includeExcludeMatcher, + IWarningCollector warningCollector) throws CompilationException { + // Prepare to retrieve the objects + List<Blob> filesOnly = new ArrayList<>(); + String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME); + Storage gcs = buildClient(configuration); + Storage.BlobListOption options = Storage.BlobListOption.prefix(ExternalDataUtils.getPrefix(configuration)); + Page<Blob> items; + + try { + items = gcs.list(container, options); + } catch (BaseServiceException ex) { + throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex)); + } + + // Collect the paths to files only + collectAndFilterFiles(items, includeExcludeMatcher.getPredicate(), includeExcludeMatcher.getMatchersList(), + filesOnly); + + // Warn if no files are returned + if (filesOnly.isEmpty() && warningCollector.shouldWarn()) { + Warning warning = Warning.of(null, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES); + warningCollector.warn(warning); + } + + return filesOnly; + } + + /** + * Excludes paths ending with "/" as that's a directory indicator, we need to return the files only + * + * @param items List of returned objects + */ + private static void collectAndFilterFiles(Page<Blob> items, BiPredicate<List<Matcher>, String> predicate, + List<Matcher> matchers, List<Blob> filesOnly) { + for (Blob item : items.iterateAll()) { + // skip folders + if (item.getName().endsWith("/")) { + continue; + } + + // No filter, add file + if (predicate.test(matchers, item.getName())) { + filesOnly.add(item); + } + } + } + + /** + * Builds the client using the provided configuration + * + * @param configuration properties + * @param numberOfPartitions number of partitions in the cluster + */ + public static void configureHdfsJobConf(JobConf conf, Map<String, String> configuration, int numberOfPartitions) { + String jsonCredentials = configuration.get(JSON_CREDENTIALS_FIELD_NAME); + String endpoint = configuration.get(ENDPOINT_FIELD_NAME); + + // disable caching FileSystem + HDFSUtils.disableHadoopFileSystemCache(conf, HADOOP_GCS_PROTOCOL); + + // TODO(htowaileb): needs further testing, recommended to disable by gcs-hadoop team + conf.set(GCSConstants.HADOOP_SUPPORT_COMPRESSED, ExternalDataConstants.FALSE); + + // TODO(htowaileb): needs further testing + // set number of threads + // conf.set(GCSConstants.HADOOP_MAX_REQUESTS_PER_BATCH, String.valueOf(numberOfPartitions)); + // conf.set(GCSConstants.HADOOP_BATCH_THREADS, String.valueOf(numberOfPartitions)); + + // authentication method + // TODO(htowaileb): find a way to pass the content instead of the path to keyfile, this line is temporary + Path credentials = Path.of("credentials.json"); + if (jsonCredentials == null) { + // anonymous access + conf.set(HADOOP_AUTH_TYPE, HADOOP_AUTH_UNAUTHENTICATED); + } else { + // TODO(htowaileb) need to pass the file content + conf.set(HADOOP_AUTH_TYPE, HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE); + conf.set(HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE_PATH, credentials.toAbsolutePath().toString()); + } + + // set endpoint if provided, default is https://storage.googleapis.com/ + if (endpoint != null) { + conf.set(HADOOP_ENDPOINT, endpoint); + } + } +} diff --cc hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitorJson.java index c0c5681f14,464d15e4b1..e54ef02434 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitorJson.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitorJson.java @@@ -103,9 -99,7 +102,8 @@@ public class LogicalOperatorPrettyPrint private static final String CONDITION_FIELD = "condition"; private static final String MISSING_VALUE_FIELD = "missing-value"; private static final String OPTIMIZER_ESTIMATES = "optimizer-estimates"; - private static final String QUERY_PLAN = "plan"; private final Map<AbstractLogicalOperator, String> operatorIdentity = new HashMap<>(); + private Map<Object, String> log2odid = Collections.emptyMap(); private final IdCounter idCounter = new IdCounter(); private final JsonGenerator jsonGenerator;
