[21/21] asterixdb git commit: [ASTERIXDB-2286][COMP][FUN][HYR] Parallel Sort Optimization

Mon, 15 Oct 2018 21:19:31 -0700 

[ASTERIXDB-2286][COMP][FUN][HYR] Parallel Sort Optimization

- user model changes: yes
- storage format changes: no
- interface changes: yes

details:
- new plan for sort operation which includes sampling and
replicating the stream of data to be sorted. Sort-merge connector
is removed from the plan. The sorted result now is in multiple partitions.
- new optimization rule to check whether full parallel sort is applicable.
- new Forward operator to read the replicated sort input stream and
to receive the ouput of the sampling.
- new sequential merge connector to merge a globally ordered result residing
in multiple partitions (in addition to the connector's partition computer).
- "asterix-lang-aql/pom.xml" is changed as a result of refactoring
code related to the range map handling.
- new private sampling function to generate the range map object
(local & global functions) & their type computers.

user model changes:
- new compiler property is added to enable and disable parallel sort.

interface changes:
- "ILogicalOperatorVisitor.java" includes Forward Operator.
- "ITuplePartitionComputer.java" includes initialize() to enable partitioner
to do some initialization. FieldRangePartitionComputerFactory uses it to
pick a range map.
- "ITuplePartitionComputerFactory.java". createPartitioner() is changed to
createPartitioner(IHyracksTaskContext hyracksTaskContext). Context is needed
for transferring the range map throught the context.

Change-Id: I73e128029a46f45e6b68c23dfb9310d5de10582f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2393
Tested-by: Jenkins <[email protected]>
Contrib: Jenkins <[email protected]>
Integration-Tests: Jenkins <[email protected]>
Reviewed-by: Dmitry Lychagin <[email protected]>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/80225e2c
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/80225e2c
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/80225e2c

Branch: refs/heads/master
Commit: 80225e2c27d77514ecaa774235951187ef524193
Parents: adfb633
Author: Ali Alsuliman <[email protected]>
Authored: Mon Oct 15 14:17:07 2018 -0700
Committer: Ali Alsuliman <[email protected]>
Committed: Mon Oct 15 21:17:44 2018 -0700

----------------------------------------------------------------------
 .../asterix/optimizer/base/RuleCollections.java |    7 +-
 .../rules/CheckFullParallelSortRule.java        |   79 +
 .../rules/FixReplicateOperatorOutputsRule.java  |    2 +-
 .../SweepIllegalNonfunctionalFunctions.java     |   19 +-
 .../subplan/InlineAllNtsInSubplanVisitor.java   |    7 +
 ...neLeftNtsInSubplanJoinFlatteningVisitor.java |   35 +-
 .../SubplanSpecialFlatteningCheckVisitor.java   |   11 +-
 .../LangExpressionToPlanTranslator.java         |    6 +-
 .../apache/asterix/api/common/APIFramework.java |   30 +-
 .../app/resource/OperatorResourcesComputer.java |    1 +
 .../app/resource/PlanStagesGenerator.java       |   90 +-
 .../app/resource/RequiredCapacityVisitor.java   |    7 +
 .../app/translator/RequestParameters.java       |    2 +-
 .../asterix-app/src/main/resources/cc.conf      |    1 +
 .../dataflow/TestPartitionComputerFactory.java  |    3 +-
 .../asterix/test/optimizer/OptimizerTest.java   |    5 +
 ...-probe-pidx-with-join-btree-sidx_01_ps.sqlpp |   76 +
 ...-probe-pidx-with-join-btree-sidx_02_ps.sqlpp |   78 +
 .../btree-index/btree-secondary-68_ps.sqlpp     |   56 +
 ...ree-sidx-idxonly-01-disable-idxonly_ps.sqlpp |   68 +
 .../btree-index/btree-sidx-idxonly-01_ps.sqlpp  |   66 +
 .../queries/introhashpartitionmerge.sqlpp       |    6 +-
 .../ngram-contains-panic_ps.sqlpp               |   51 +
 .../ngram-contains_ps.sqlpp                     |   51 +
 .../olist-edit-distance-check-panic_ps.sqlpp    |   63 +
 .../olist-edit-distance-check_ps.sqlpp          |   63 +
 .../olist-edit-distance-panic_ps.sqlpp          |   63 +
 .../olist-edit-distance_ps.sqlpp                |   63 +
 .../olist-fuzzyeq-edit-distance_ps.sqlpp        |   65 +
 .../inverted-index-basic/word-contains_ps.sqlpp |   51 +
 ...olist-edit-distance-check-let-panic_ps.sqlpp |   65 +
 .../olist-edit-distance-check-let_ps.sqlpp      |   65 +
 ...ith-join-edit-distance-check-idx_01_ps.sqlpp |   81 +
 ...pidx-with-join-jaccard-check-idx_01_ps.sqlpp |   81 +
 .../inverted-index-join/ngram-contains_ps.sqlpp |   52 +
 .../optimizerts/queries/loj-core_ps.sqlpp       |   52 +
 .../optimizerts/queries/loj-sugar_ps.sqlpp      |   48 +
 ...-probe-pidx-with-join-btree-sidx_01_ps.sqlpp |   75 +
 ...-probe-pidx-with-join-btree-sidx_02_ps.sqlpp |   75 +
 .../ngram-contains-panic_ps.sqlpp               |   56 +
 .../ngram-contains_ps.sqlpp                     |   56 +
 .../olist-edit-distance-check-panic_ps.sqlpp    |   70 +
 .../olist-edit-distance-check_ps.sqlpp          |   70 +
 .../olist-edit-distance-panic_ps.sqlpp          |   70 +
 .../olist-edit-distance_ps.sqlpp                |   70 +
 .../olist-fuzzyeq-edit-distance_ps.sqlpp        |   70 +
 .../inverted-index-basic/word-contains_ps.sqlpp |   56 +
 ...olist-edit-distance-check-let-panic_ps.sqlpp |   72 +
 .../olist-edit-distance-check-let_ps.sqlpp      |   72 +
 ...ith-join-edit-distance-check-idx_01_ps.sqlpp |   76 +
 ...pidx-with-join-jaccard-check-idx_01_ps.sqlpp |   76 +
 ...-probe-pidx-with-join-rtree-sidx_01_ps.sqlpp |   82 +
 ...-probe-pidx-with-join-rtree-sidx_02_ps.sqlpp |   82 +
 ...robe-pidx-with-join-btree-sidx_01_1_ps.sqlpp |   74 +
 ...robe-pidx-with-join-btree-sidx_01_2_ps.sqlpp |   75 +
 ...robe-pidx-with-join-btree-sidx_02_1_ps.sqlpp |   74 +
 ...robe-pidx-with-join-btree-sidx_02_2_ps.sqlpp |   75 +
 .../ngram-contains-panic_ps.sqlpp               |   55 +
 .../ngram-contains_ps.sqlpp                     |   55 +
 .../inverted-index-basic/word-contains_ps.sqlpp |   55 +
 ...ith-join-edit-distance-check-idx_01_ps.sqlpp |   75 +
 .../ngram-contains_01_ps.sqlpp                  |   61 +
 .../ngram-contains_02_ps.sqlpp                  |   61 +
 .../ngram-contains_03_ps.sqlpp                  |   51 +
 .../ngram-contains_04_ps.sqlpp                  |   62 +
 ...-probe-pidx-with-join-rtree-sidx_01_ps.sqlpp |   81 +
 ...-probe-pidx-with-join-rtree-sidx_02_ps.sqlpp |   81 +
 ...robe-pidx-with-join-btree-sidx_01_1_ps.sqlpp |   69 +
 ...robe-pidx-with-join-btree-sidx_01_2_ps.sqlpp |   70 +
 ...robe-pidx-with-join-btree-sidx_02_1_ps.sqlpp |   69 +
 ...robe-pidx-with-join-btree-sidx_02_2_ps.sqlpp |   70 +
 .../ngram-contains-panic_ps.sqlpp               |   50 +
 .../ngram-contains_ps.sqlpp                     |   50 +
 .../inverted-index-basic/word-contains_ps.sqlpp |   50 +
 ...ith-join-edit-distance-check-idx_01_ps.sqlpp |   70 +
 .../ngram-contains_01_ps.sqlpp                  |   61 +
 .../ngram-contains_02_ps.sqlpp                  |   61 +
 .../ngram-contains_03_ps.sqlpp                  |   51 +
 .../ngram-contains_04_ps.sqlpp                  |   62 +
 ...-probe-pidx-with-join-rtree-sidx_01_ps.sqlpp |   76 +
 ...-probe-pidx-with-join-rtree-sidx_02_ps.sqlpp |   76 +
 .../btree-equi-join-non-enforced-05_ps.sqlpp    |   46 +
 .../btree-equi-join-non-enforced-06_ps.sqlpp    |   46 +
 .../btree-equi-join-non-enforced-07_ps.sqlpp    |   47 +
 .../btree-equi-join-non-enforced-08_ps.sqlpp    |   47 +
 .../btree-equi-join-non-enforced-09_ps.sqlpp    |   46 +
 .../btree-index-non-enforced-04_ps.sqlpp        |   44 +
 .../btree-index-non-enforced-05_ps.sqlpp        |   44 +
 .../btree-index-non-enforced-06_ps.sqlpp        |   44 +
 .../btree-index-non-enforced-07_ps.sqlpp        |   44 +
 .../btree-index-non-enforced-08_ps.sqlpp        |   44 +
 .../btree-index-non-enforced-09_ps.sqlpp        |   44 +
 .../btree-index-non-enforced-105_ps.sqlpp       |   46 +
 .../btree-index-non-enforced-10_ps.sqlpp        |   44 +
 .../btree-index-non-enforced-11_ps.sqlpp        |   44 +
 .../queries/orderby-desc-using-gby_ps.sqlpp     |   59 +
 ...rs-index-search-conjunctive-open_01_ps.sqlpp |   56 +
 ...rs-index-search-conjunctive-open_02_ps.sqlpp |   56 +
 .../orders-index-search-conjunctive_01_ps.sqlpp |   56 +
 .../orders-index-search-conjunctive_02_ps.sqlpp |   56 +
 .../parallel_sort_enabled_disabled.1.sqlpp      |   40 +
 .../parallel_sort_enabled_disabled.2.sqlpp      |   40 +
 .../q01_pricing_summary_report_nt_ps.sqlpp      |   80 +
 .../queries/query-ASTERIXDB-1806_ps.sqlpp       |   66 +
 .../queries/query-ASTERIXDB-2354_ps.sqlpp       |   29 +
 .../queries/query-ASTERIXDB-2408_ps.sqlpp       |   45 +
 .../queries/query-ASTERIXDB-810-2_ps.sqlpp      |   70 +
 .../queries/query-ASTERIXDB-810-3_ps.sqlpp      |   70 +
 .../queries/query-ASTERIXDB-810_ps.sqlpp        |   70 +
 .../optimizerts/queries/query-issue562_ps.sqlpp |  108 +
 ...-probe-pidx-with-join-rtree-sidx_01_ps.sqlpp |   77 +
 ...-probe-pidx-with-join-rtree-sidx_02_ps.sqlpp |   77 +
 .../skip-index/skip-ngram-index_ps.sqlpp        |   51 +
 .../queries/split-materialization_ps.sqlpp      |   61 +
 .../queries/subquery/exists_ps.sqlpp            |   72 +
 .../queries/subquery/in_as_or_1_ps.sqlpp        |   42 +
 .../queries/subquery/in_as_or_2_ps.sqlpp        |   43 +
 .../queries/subquery/in_as_or_3_ps.sqlpp        |   46 +
 .../queries/subquery/in_as_or_4_ps.sqlpp        |   47 +
 .../queries/subquery/in_as_or_5_ps.sqlpp        |   48 +
 .../queries/subquery/in_as_or_6_ps.sqlpp        |   49 +
 .../queries/subquery/in_correlated_ps.sqlpp     |   44 +
 .../optimizerts/queries/subquery/in_ps.sqlpp    |   44 +
 .../queries/subquery/not_exists_ps.sqlpp        |   73 +
 .../queries/tpcds/query-ASTERIXDB-1596_ps.sqlpp |  131 +
 .../tpch/q12_shipping_broadcast_ps.sqlpp        |   53 +
 .../queries/tpch/q12_shipping_ps.sqlpp          |   53 +
 ...n-probe-pidx-with-join-btree-sidx_01_ps.plan |  109 +
 ...n-probe-pidx-with-join-btree-sidx_02_ps.plan |   77 +
 .../secondary-self-equi-join-index-only.plan    |   16 +-
 ...on-idxonly-to-sidx-idxonly-equi-join_01.plan |   50 +-
 .../btree-index/btree-secondary-68_ps.plan      |   69 +
 ...tree-sidx-idxonly-01-disable-idxonly_ps.plan |   47 +
 .../btree-index/btree-sidx-idxonly-01_ps.plan   |   65 +
 .../results/hints/fullparallelsort.plan         |  117 +
 .../results/hints/nofullparallel_hint.plan      |   38 +
 .../ngram-contains-panic_ps.plan                |   28 +
 .../inverted-index-basic/ngram-contains_ps.plan |   38 +
 .../olist-edit-distance-check-panic_ps.plan     |   28 +
 .../olist-edit-distance-check_ps.plan           |   38 +
 .../olist-edit-distance-panic_ps.plan           |   28 +
 .../olist-edit-distance_ps.plan                 |   38 +
 .../olist-fuzzyeq-edit-distance_ps.plan         |   38 +
 .../inverted-index-basic/word-contains_ps.plan  |   28 +
 .../olist-edit-distance-check-let-panic_ps.plan |   28 +
 .../olist-edit-distance-check-let_ps.plan       |   38 +
 ...with-join-edit-distance-check-idx_01_ps.plan |  165 +
 ...-pidx-with-join-jaccard-check-idx_01_ps.plan |  353 +
 .../inverted-index-join/ngram-contains_ps.plan  |   54 +
 .../optimizerts/results/loj-core_ps.plan        |   53 +
 .../optimizerts/results/loj-sugar_ps.plan       |   53 +
 ...n-probe-pidx-with-join-btree-sidx_01_ps.plan |   86 +
 ...n-probe-pidx-with-join-btree-sidx_02_ps.plan |   86 +
 .../ngram-contains-panic_ps.plan                |   32 +
 .../inverted-index-basic/ngram-contains_ps.plan |   42 +
 .../olist-edit-distance-check-panic_ps.plan     |   32 +
 .../olist-edit-distance-check_ps.plan           |   42 +
 .../olist-edit-distance-panic_ps.plan           |   32 +
 .../olist-edit-distance_ps.plan                 |   42 +
 .../olist-fuzzyeq-edit-distance_ps.plan         |   42 +
 .../inverted-index-basic/word-contains_ps.plan  |   32 +
 .../olist-edit-distance-check-let-panic_ps.plan |   32 +
 .../olist-edit-distance-check-let_ps.plan       |   42 +
 ...with-join-edit-distance-check-idx_01_ps.plan |  190 +
 ...-pidx-with-join-jaccard-check-idx_01_ps.plan |  358 +
 ...n-probe-pidx-with-join-rtree-sidx_01_ps.plan |   90 +
 ...n-probe-pidx-with-join-rtree-sidx_02_ps.plan |   92 +
 ...probe-pidx-with-join-btree-sidx_01_1_ps.plan |   86 +
 ...probe-pidx-with-join-btree-sidx_01_2_ps.plan |   86 +
 ...probe-pidx-with-join-btree-sidx_02_1_ps.plan |   86 +
 ...probe-pidx-with-join-btree-sidx_02_2_ps.plan |   86 +
 .../ngram-contains-panic_ps.plan                |   32 +
 .../inverted-index-basic/ngram-contains_ps.plan |   42 +
 .../inverted-index-basic/word-contains_ps.plan  |   32 +
 ...with-join-edit-distance-check-idx_01_ps.plan |  190 +
 .../ngram-contains_01_ps.plan                   |   50 +
 .../ngram-contains_02_ps.plan                   |   50 +
 .../ngram-contains_03_ps.plan                   |   54 +
 .../ngram-contains_04_ps.plan                   |   54 +
 ...n-probe-pidx-with-join-rtree-sidx_01_ps.plan |   90 +
 ...n-probe-pidx-with-join-rtree-sidx_02_ps.plan |   92 +
 ...probe-pidx-with-join-btree-sidx_01_1_ps.plan |   77 +
 ...probe-pidx-with-join-btree-sidx_01_2_ps.plan |   77 +
 ...probe-pidx-with-join-btree-sidx_02_1_ps.plan |   77 +
 ...probe-pidx-with-join-btree-sidx_02_2_ps.plan |   77 +
 .../ngram-contains-panic_ps.plan                |   28 +
 .../inverted-index-basic/ngram-contains_ps.plan |   38 +
 .../inverted-index-basic/word-contains_ps.plan  |   28 +
 ...with-join-edit-distance-check-idx_01_ps.plan |  165 +
 .../ngram-contains_01_ps.plan                   |   50 +
 .../ngram-contains_02_ps.plan                   |   54 +
 .../ngram-contains_03_ps.plan                   |   54 +
 .../ngram-contains_04_ps.plan                   |   54 +
 ...n-probe-pidx-with-join-rtree-sidx_01_ps.plan |   83 +
 ...n-probe-pidx-with-join-rtree-sidx_02_ps.plan |   83 +
 .../btree-equi-join-non-enforced-05_ps.plan     |   56 +
 .../btree-equi-join-non-enforced-06_ps.plan     |   56 +
 .../btree-equi-join-non-enforced-07_ps.plan     |   56 +
 .../btree-equi-join-non-enforced-08_ps.plan     |   58 +
 .../btree-equi-join-non-enforced-09_ps.plan     |   58 +
 .../btree-index-non-enforced-04_ps.plan         |   48 +
 .../btree-index-non-enforced-05_ps.plan         |   48 +
 .../btree-index-non-enforced-06_ps.plan         |   48 +
 .../btree-index-non-enforced-07_ps.plan         |   48 +
 .../btree-index-non-enforced-08_ps.plan         |   48 +
 .../btree-index-non-enforced-09_ps.plan         |   48 +
 .../btree-index-non-enforced-105_ps.plan        |   70 +
 .../btree-index-non-enforced-10_ps.plan         |   48 +
 .../btree-index-non-enforced-11_ps.plan         |   50 +
 .../results/orderby-desc-using-gby_ps.plan      |   44 +
 ...ers-index-search-conjunctive-open_01_ps.plan |   60 +
 ...ers-index-search-conjunctive-open_02_ps.plan |   60 +
 .../orders-index-search-conjunctive_01_ps.plan  |   60 +
 .../orders-index-search-conjunctive_02_ps.plan  |   60 +
 .../parallel_sort_enabled_disabled.1.plan       |   30 +
 .../parallel_sort_enabled_disabled.2.plan       |   12 +
 .../q01_pricing_summary_report_nt_ps.plan       |   74 +
 .../results/query-ASTERIXDB-1806_ps.plan        |   59 +
 .../results/query-ASTERIXDB-2354_ps.plan        |   98 +
 .../results/query-ASTERIXDB-2408_ps.plan        |   72 +
 .../results/query-ASTERIXDB-810-2_ps.plan       |   83 +
 .../results/query-ASTERIXDB-810-3_ps.plan       |   83 +
 .../results/query-ASTERIXDB-810_ps.plan         |   83 +
 .../optimizerts/results/query-issue562_ps.plan  |  135 +
 .../rtree-index-join/issue730-index-only.plan   |   22 +-
 ...n-probe-pidx-with-join-rtree-sidx_01_ps.plan |  133 +
 ...n-probe-pidx-with-join-rtree-sidx_02_ps.plan |   83 +
 ...spatial-self-intersect-point-index-only.plan |   16 +-
 .../results/skip-index/skip-ngram-index_ps.plan |   28 +
 .../results/split-materialization_ps.plan       |   67 +
 .../optimizerts/results/subquery/exists_ps.plan |  157 +
 .../results/subquery/in_as_or_1_ps.plan         |   41 +
 .../results/subquery/in_as_or_2_ps.plan         |   67 +
 .../results/subquery/in_as_or_3_ps.plan         |   41 +
 .../results/subquery/in_as_or_4_ps.plan         |   67 +
 .../results/subquery/in_as_or_5_ps.plan         |   41 +
 .../results/subquery/in_as_or_6_ps.plan         |   67 +
 .../results/subquery/in_correlated_ps.plan      |   84 +
 .../optimizerts/results/subquery/in_ps.plan     |   74 +
 .../results/subquery/not_exists_ps.plan         |  157 +
 .../results/tpcds/query-ASTERIXDB-1596_ps.plan  |   58 +
 .../results/tpch/q12_shipping_broadcast_ps.plan |   95 +
 .../results/tpch/q12_shipping_ps.plan           |   95 +
 .../big_object_insert.3.query.aql               |    4 +-
 .../prefix-search/prefix-search.3.query.aql     |    2 +-
 .../misc/stable_sort/stable_sort.3.query.aql    |    2 +-
 .../overlap_bins_gby_0.3.query.aql              |    4 +-
 .../p_sort_num_samples.1.ddl.sqlpp              |   47 +
 .../p_sort_num_samples.2.update.sqlpp           |   22 +
 .../p_sort_num_samples.3.query.sqlpp            |   26 +
 .../p_sort_seq_merge.1.ddl.sqlpp                |   36 +
 .../p_sort_seq_merge.2.update.sqlpp             |   42 +
 .../p_sort_seq_merge.3.query.sqlpp              |   24 +
 .../prefix-search/prefix-search.3.query.sqlpp   |    2 +-
 .../misc/stable_sort/stable_sort.3.query.sqlpp  |    3 +-
 .../object_remove/object_remove.3.query.sqlpp   |    2 +-
 .../single_dataset_with_index.13.query.sqlpp    |    2 +
 .../single_dataset_with_index.8.query.sqlpp     |    2 +
 .../overlap_bins_gby_0.3.query.sqlpp            |    2 +-
 .../cluster_state_1/cluster_state_1.1.regexadm  |    2 +
 .../cluster_state_1_full.1.regexadm             |    2 +
 .../cluster_state_1_less.1.regexadm             |    2 +
 .../p_sort_num_samples/p_sort_num_samples.3.adm | 6005 ++++++++++++++++
 .../p_sort_seq_merge/p_sort_seq_merge.3.adm     |    1 +
 .../misc/prefix-search/prefix-search.1.adm      |   90 +-
 .../results/misc/stable_sort/stable_sort.3.adm  | 6638 +++++++++---------
 .../misc/prefix-search/prefix-search.3.ast      |    5 +
 .../misc/stable_sort/stable_sort.3.ast          |   10 +
 .../overlap_bins_gby_0/overlap_bins_gby_0.3.ast |   10 +
 .../resources/runtimets/testsuite_sqlpp.xml     |   10 +
 .../common/config/CompilerProperties.java       |   21 +-
 .../asterix/common/exceptions/ErrorCode.java    |    1 +
 .../main/resources/asx_errormsg/en.properties   |    1 +
 .../ByteArrayAccessibleDataInputStream.java     |   33 -
 .../base/ByteArrayAccessibleInputStream.java    |   48 -
 asterixdb/asterix-lang-aql/pom.xml              |    8 -
 .../asterix/lang/aql/util/RangeMapBuilder.java  |  177 -
 .../asterix-lang-aql/src/main/javacc/AQL.jj     |    9 +-
 asterixdb/asterix-lang-common/pom.xml           |    4 +
 .../lang/common/clause/OrderbyClause.java       |   13 +-
 .../lang/common/util/RangeMapBuilder.java       |  175 +
 .../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj |   14 +-
 .../asterix/om/functions/BuiltinFunctions.java  |   15 +
 .../impl/ListOfSamplesTypeComputer.java         |   45 +
 .../std/LocalSamplingAggregateDescriptor.java   |  173 +
 .../std/RangeMapAggregateDescriptor.java        |  317 +
 .../runtime/functions/FunctionCollection.java   |    4 +
 .../runtime/functions/FunctionTypeInferers.java |   26 +-
 .../core/algebra/base/LogicalOperatorTag.java   |    1 +
 .../core/algebra/base/OperatorAnnotations.java  |    3 +-
 .../core/algebra/base/PhysicalOperatorTag.java  |    2 +
 .../AbstractFunctionCallExpression.java         |   19 +-
 .../AggregateFunctionCallExpression.java        |    2 +
 .../logical/AbstractReplicateOperator.java      |   16 +-
 .../operators/logical/ForwardOperator.java      |  111 +
 .../visitors/CardinalityInferenceVisitor.java   |    8 +-
 .../visitors/FDsAndEquivClassesVisitor.java     |    7 +
 .../visitors/IsomorphismOperatorVisitor.java    |   13 +
 .../IsomorphismVariableMappingVisitor.java      |    7 +
 ...OperatorDeepCopyWithNewVariablesVisitor.java |    9 +
 .../visitors/LogicalPropertiesVisitor.java      |    9 +-
 .../visitors/OperatorDeepCopyVisitor.java       |    6 +
 .../visitors/PrimaryKeyVariablesVisitor.java    |    8 +-
 .../visitors/ProducedVariableVisitor.java       |    8 +-
 .../logical/visitors/SchemaVariableVisitor.java |    8 +
 .../visitors/SubstituteVariableVisitor.java     |    9 +
 .../logical/visitors/UsedVariableVisitor.java   |   54 +-
 .../physical/AbstractStableSortPOperator.java   |   54 +-
 .../operators/physical/ForwardPOperator.java    |  136 +
 .../RangePartitionExchangePOperator.java        |   55 +-
 .../RangePartitionMergeExchangePOperator.java   |   10 +-
 .../SequentialMergeExchangePOperator.java       |   76 +
 .../LogicalOperatorPrettyPrintVisitor.java      |    8 +
 .../LogicalOperatorPrettyPrintVisitorJson.java  |    9 +
 .../algebra/properties/LocalOrderProperty.java  |   15 +-
 .../core/algebra/properties/PropertiesUtil.java |   61 +-
 .../visitors/ILogicalOperatorVisitor.java       |    2 +
 .../core/config/AlgebricksConfig.java           |    2 +
 .../impl/ConnectorPolicyAssignmentPolicy.java   |    4 +-
 .../core/rewriter/base/HeuristicOptimizer.java  |   37 +-
 .../base/PhysicalOptimizationConfig.java        |   33 +-
 .../core/utils/DotFormatGenerator.java          |  152 +-
 .../core/utils/LogicalOperatorDotVisitor.java   |  390 +-
 .../rules/EnforceStructuralPropertiesRule.java  |  370 +-
 .../rules/ExtractCommonOperatorsRule.java       |   66 +
 .../rewriter/rules/InlineVariablesRule.java     |   30 +-
 .../SetAlgebricksPhysicalOperatorsRule.java     |    4 +
 ...placeNtsWithSubplanInputOperatorVisitor.java |    8 +-
 .../dataflow/value/ITuplePartitionComputer.java |   18 +
 .../value/ITuplePartitionComputerFactory.java   |    4 +-
 .../hyracks/api/exceptions/ErrorCode.java       |    3 +
 .../src/main/resources/errormsg/en.properties   |    3 +
 .../ByteArrayAccessibleDataInputStream.java     |   33 +
 .../util/ByteArrayAccessibleInputStream.java    |   48 +
 .../FieldHashPartitionComputerFactory.java      |    3 +-
 .../partition/OnePartitionComputerFactory.java  |   38 +
 .../RandomPartitionComputerFactory.java         |    3 +-
 .../partition/RepartitionComputerFactory.java   |    5 +-
 ...namicFieldRangePartitionComputerFactory.java |   48 +
 .../FieldRangePartitionComputerFactory.java     |   44 +-
 .../common/data/partition/range/IRangeMap.java  |   35 -
 .../common/data/partition/range/RangeMap.java   |  116 +-
 ...taticFieldRangePartitionComputerFactory.java |   38 +
 .../base/AbstractMToNConnectorDescriptor.java   |   18 +
 .../AbstractReplicateOperatorDescriptor.java    |    1 +
 .../DeterministicPartitionBatchManager.java     |   80 +
 .../collectors/SequentialMergeFrameReader.java  |   77 +
 ...wareMToNPartitioningConnectorDescriptor.java |    2 +-
 .../MToNBroadcastConnectorDescriptor.java       |   16 -
 .../MToNPartitioningConnectorDescriptor.java    |   19 +-
 ...NPartitioningMergingConnectorDescriptor.java |    2 +-
 ...titioningWithMessageConnectorDescriptor.java |    2 +-
 ...OneSequentialMergingConnectorDescriptor.java |   65 +
 .../std/connectors/PartitionDataWriter.java     |    1 +
 .../join/HybridHashJoinOperatorDescriptor.java  |   12 +-
 .../InMemoryHashJoinOperatorDescriptor.java     |    4 +-
 .../std/misc/ForwardOperatorDescriptor.java     |  246 +
 .../std/sort/AbstractExternalSortRunMerger.java |    9 +-
 358 files changed, 27634 insertions(+), 4310 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
index 3c981d4..1010a84 100644
--- 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
+++ 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
@@ -24,6 +24,7 @@ import java.util.LinkedList;
 import java.util.List;
 
 import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.om.functions.BuiltinFunctions;
 import 
org.apache.asterix.optimizer.rules.AddEquivalenceClassForRecordConstructorRule;
 import 
org.apache.asterix.optimizer.rules.AsterixExtractFunctionsFromJoinConditionRule;
 import org.apache.asterix.optimizer.rules.AsterixInlineVariablesRule;
@@ -31,6 +32,7 @@ import 
org.apache.asterix.optimizer.rules.AsterixIntroduceGroupByCombinerRule;
 import org.apache.asterix.optimizer.rules.ByNameToByIndexFieldAccessRule;
 import org.apache.asterix.optimizer.rules.CancelUnnestWithNestedListifyRule;
 import org.apache.asterix.optimizer.rules.CheckFilterExpressionTypeRule;
+import org.apache.asterix.optimizer.rules.CheckFullParallelSortRule;
 import org.apache.asterix.optimizer.rules.CheckInsertUpsertReturningRule;
 import org.apache.asterix.optimizer.rules.ConstantFoldingRule;
 import org.apache.asterix.optimizer.rules.CountVarToCountOneRule;
@@ -338,7 +340,9 @@ public final class RuleCollections {
         physicalRewritesAllLevels.add(new 
SetAlgebricksPhysicalOperatorsRule());
         physicalRewritesAllLevels.add(new SetAsterixPhysicalOperatorsRule());
         physicalRewritesAllLevels.add(new 
AddEquivalenceClassForRecordConstructorRule());
-        physicalRewritesAllLevels.add(new EnforceStructuralPropertiesRule());
+        physicalRewritesAllLevels.add(new CheckFullParallelSortRule());
+        physicalRewritesAllLevels
+                .add(new 
EnforceStructuralPropertiesRule(BuiltinFunctions.RANGE_MAP, 
BuiltinFunctions.LOCAL_SAMPLING));
         physicalRewritesAllLevels.add(new RemoveSortInFeedIngestionRule());
         physicalRewritesAllLevels.add(new 
RemoveUnnecessarySortMergeExchange());
         physicalRewritesAllLevels.add(new PushProjectDownRule());
@@ -377,6 +381,7 @@ public final class RuleCollections {
         prepareForJobGenRewrites.add(new SetAlgebricksPhysicalOperatorsRule());
         prepareForJobGenRewrites
                 .add(new 
IsolateHyracksOperatorsRule(HeuristicOptimizer.hyraxOperatorsBelowWhichJobGenIsDisabled));
+        prepareForJobGenRewrites.add(new FixReplicateOperatorOutputsRule());
         prepareForJobGenRewrites.add(new ExtractCommonOperatorsRule());
         // Re-infer all types, so that, e.g., the effect of not-is-null is
         // propagated.

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFullParallelSortRule.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFullParallelSortRule.java
 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFullParallelSortRule.java
new file mode 100644
index 0000000..7b13a2b
--- /dev/null
+++ 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFullParallelSortRule.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
+import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * <pre>
+ * Description:
+ *      This rule checks whether full parallel sort is applicable to {@link 
OrderOperator}. It disables full parallel
+ *      sort when a limit operator or running aggregate operator is present in 
the parents of the order operator.
+ * Pre-conditions:
+ *      None.
+ * Post-requirements:
+ *      1. {@link 
org.apache.hyracks.algebricks.rewriter.rules.EnforceStructuralPropertiesRule}
+ * </pre>
+ */
+public class CheckFullParallelSortRule implements IAlgebraicRewriteRule {
+    private final List<AbstractLogicalOperator> parents = new ArrayList<>();
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, 
IOptimizationContext context)
+            throws AlgebricksException {
+        parents.add((AbstractLogicalOperator) opRef.getValue());
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, 
IOptimizationContext ctx) throws AlgebricksException {
+        parents.remove(parents.size() - 1);
+        AbstractLogicalOperator orderOp = (AbstractLogicalOperator) 
opRef.getValue();
+        if (orderOp.getOperatorTag() == LogicalOperatorTag.ORDER
+                && 
!orderOp.getAnnotations().containsKey(OperatorAnnotations.USE_STATIC_RANGE)) {
+            // disable when sort output is consumed by limit & running agg op 
(result should be unpartitioned for limit)
+            AbstractLogicalOperator parent;
+            for (int i = parents.size() - 1; i >= 0; i--) {
+                parent = parents.get(i);
+                if (parent.getOperatorTag() == LogicalOperatorTag.LIMIT
+                        && ((LimitOperator) parent).isTopmostLimitOp()) {
+                    
orderOp.getAnnotations().put(OperatorAnnotations.USE_DYNAMIC_RANGE, 
Boolean.FALSE);
+                    return true;
+                }
+                if (parent.getOperatorTag() == 
LogicalOperatorTag.RUNNINGAGGREGATE) {
+                    
orderOp.getAnnotations().put(OperatorAnnotations.USE_DYNAMIC_RANGE, 
Boolean.FALSE);
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FixReplicateOperatorOutputsRule.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FixReplicateOperatorOutputsRule.java
 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FixReplicateOperatorOutputsRule.java
index bd79a38..dc135c9 100644
--- 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FixReplicateOperatorOutputsRule.java
+++ 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FixReplicateOperatorOutputsRule.java
@@ -140,7 +140,7 @@ public class FixReplicateOperatorOutputsRule implements 
IAlgebraicRewriteRule {
             boolean parentFixed = false;
             for (int oldParentIndex = 0; oldParentIndex < 
replicateOperator.getOutputs().size(); oldParentIndex++) {
                 if 
(parentsPathToReplicate.contains(replicateOperator.getOutputs().get(oldParentIndex)))
 {
-                    
replicateOperator.getOutputs().get(oldParentIndex).setValue(replicateActualParent.getValue());
+                    replicateOperator.getOutputs().set(oldParentIndex, 
replicateActualParent);
                     parentFixed = true;
                     updateNumberOfParentsDone(replicateOperator);
                     break;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
index 8372851..0c91e9b 100644
--- 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
+++ 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
@@ -34,11 +34,12 @@ import 
org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCa
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOperator;
+import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.ForwardOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -101,12 +102,10 @@ public class SweepIllegalNonfunctionalFunctions extends 
AbstractExtractExprRule
     private class IllegalNonfunctionalFunctionSweeperOperatorVisitor 
implements ILogicalOperatorVisitor<Void, Void> {
 
         private void sweepExpression(ILogicalExpression expr, ILogicalOperator 
op) throws AlgebricksException {
-            if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) 
{
-                if (!expr.isFunctional()) {
-                    AbstractFunctionCallExpression fce = 
(AbstractFunctionCallExpression) expr;
-                    throw new 
CompilationException(ErrorCode.COMPILATION_ERROR, fce.getSourceLocation(),
-                            "Found non-functional function " + 
fce.getFunctionIdentifier() + " in op " + op);
-                }
+            if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL 
&& !expr.isFunctional()) {
+                AbstractFunctionCallExpression fce = 
(AbstractFunctionCallExpression) expr;
+                throw new CompilationException(ErrorCode.COMPILATION_ERROR, 
fce.getSourceLocation(),
+                        "Found non-functional function " + 
fce.getFunctionIdentifier() + " in op " + op);
             }
         }
 
@@ -308,6 +307,12 @@ public class SweepIllegalNonfunctionalFunctions extends 
AbstractExtractExprRule
         public Void visitTokenizeOperator(TokenizeOperator op, Void tag) 
throws AlgebricksException {
             return null;
         }
+
+        @Override
+        public Void visitForwardOperator(ForwardOperator op, Void arg) throws 
AlgebricksException {
+            sweepExpression(op.getRangeMapExpression().getValue(), op);
+            return null;
+        }
     }
 
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
index 4d646f3..22ef303 100644
--- 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
+++ 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
@@ -59,6 +59,7 @@ import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOper
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.ForwardOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
@@ -642,6 +643,12 @@ class InlineAllNtsInSubplanVisitor implements 
IQueryOperatorVisitor<ILogicalOper
         return visitSingleInputOperator(op);
     }
 
+    @Override
+    public ILogicalOperator visitForwardOperator(ForwardOperator op, Void arg) 
throws AlgebricksException {
+        throw new CompilationException(ErrorCode.COMPILATION_ERROR, 
op.getSourceLocation(),
+                "Forward operator should have been disqualified for this 
rewriting!");
+    }
+
     /**
      * Wraps an AggregateOperator or RunningAggregateOperator with a group-by
      * operator where the group-by keys are variables in keyVarsToEnforce. Note

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
index 610fea2..b862a6f 100644
--- 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
+++ 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
@@ -42,6 +42,7 @@ import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOper
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.ForwardOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
@@ -52,6 +53,7 @@ import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperato
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
@@ -63,23 +65,19 @@ import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOper
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
 import 
org.apache.hyracks.algebricks.core.algebra.visitors.IQueryOperatorVisitor;
 
-/*
-    This visitor inlines the input <code>nts</code> in the query plan rooted
-    at the operator being visited, with the query plan rooted at the input
-    <code>subplanInputOperator</code>.
-
-    The visitor ensures that:
-    1. live variables at <code>subplanInputOperator</code> are
-    propagated to the top-most join operator in the query plan rooted
-    at the operator being visited;
-    2. no available tuple at <code>subplanInputOperator</code> get lost along 
the
-    pipeline to the top-most join operator in the query plan rooted
-    at the operator being visited.
-*/
+/**
+ * This visitor inlines the input {@code nts} in the query plan rooted at the 
operator being visited,
+ * with the query plan rooted at the input {@code subplanInputOperator}.
+ *
+ * The visitor ensures that:
+ * 1. live variables at {@code subplanInputOperator} are propagated to the 
top-most join operator in the query plan
+ * rooted at the operator being visited.
+ * 2. no available tuple at {@code subplanInputOperator} get lost along the 
pipeline to the top-most join operator
+ * in the query plan rooted at the operator being visited.
+ */
 class InlineLeftNtsInSubplanJoinFlatteningVisitor implements 
IQueryOperatorVisitor<ILogicalOperator, Void> {
     // The optimization context.
     private final IOptimizationContext context;
@@ -380,6 +378,12 @@ class InlineLeftNtsInSubplanJoinFlatteningVisitor 
implements IQueryOperatorVisit
         return visitSingleInputOperator(op);
     }
 
+    @Override
+    public ILogicalOperator visitForwardOperator(ForwardOperator op, Void arg) 
throws AlgebricksException {
+        throw new UnsupportedOperationException(
+                "Nested subplans with a forward operator should have been 
disqualified for this rewriting!");
+    }
+
     private ILogicalOperator visitSingleInputOperator(ILogicalOperator op) 
throws AlgebricksException {
         if (op.getInputs().size() == 1) {
             // Deals with single input operators.
@@ -395,8 +399,7 @@ class InlineLeftNtsInSubplanJoinFlatteningVisitor 
implements IQueryOperatorVisit
     }
 
     /**
-     * Inject varaibles to indicate non-matches for the right branch of
-     * a left-outer join.
+     * Inject variables to indicate non-matches for the right branch of a 
left-outer join.
      *
      * @param joinOp
      *            the leftouter join operator.

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
index 44bfbe4..e2b104d 100644
--- 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
+++ 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
@@ -18,16 +18,19 @@
  */
 package org.apache.asterix.optimizer.rules.subplan;
 
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
 import org.apache.commons.lang3.mutable.Mutable;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOperator;
+import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.ForwardOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
@@ -229,6 +232,12 @@ class SubplanSpecialFlatteningCheckVisitor implements 
IQueryOperatorVisitor<Bool
         return visitInputs(op);
     }
 
+    @Override
+    public Boolean visitForwardOperator(ForwardOperator op, Void arg) throws 
AlgebricksException {
+        throw new CompilationException(ErrorCode.COMPILATION_ERROR, 
op.getSourceLocation(),
+                "Forward operator should have been disqualified for this 
rewriting!");
+    }
+
     private boolean visitInputs(ILogicalOperator op) throws 
AlgebricksException {
         for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
             if (childRef.getValue().accept(this, null)) {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index 91de474..19fe02d 100644
--- 
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ 
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -38,7 +38,6 @@ import 
org.apache.asterix.common.exceptions.CompilationException;
 import org.apache.asterix.common.exceptions.ErrorCode;
 import org.apache.asterix.common.functions.FunctionConstants;
 import org.apache.asterix.common.functions.FunctionSignature;
-import org.apache.asterix.lang.aql.util.RangeMapBuilder;
 import org.apache.asterix.lang.common.base.Expression;
 import org.apache.asterix.lang.common.base.Expression.Kind;
 import org.apache.asterix.lang.common.base.ILangExpression;
@@ -69,6 +68,7 @@ import org.apache.asterix.lang.common.struct.Identifier;
 import org.apache.asterix.lang.common.struct.OperatorType;
 import org.apache.asterix.lang.common.struct.QuantifiedPair;
 import org.apache.asterix.lang.common.util.FunctionUtil;
+import org.apache.asterix.lang.common.util.RangeMapBuilder;
 import 
org.apache.asterix.lang.common.visitor.base.AbstractQueryExpressionVisitor;
 import org.apache.asterix.metadata.MetadataManager;
 import org.apache.asterix.metadata.declared.DataSource;
@@ -1163,9 +1163,9 @@ class LangExpressionToPlanTranslator
         }
         if (oc.getRangeMap() != null) {
             Iterator<OrderModifier> orderModifIter = 
oc.getModifierList().iterator();
-            boolean ascending = (orderModifIter.next() == OrderModifier.ASC);
+            boolean ascending = orderModifIter.next() == OrderModifier.ASC;
             RangeMapBuilder.verifyRangeOrder(oc.getRangeMap(), ascending);
-            ord.getAnnotations().put(OperatorAnnotations.USE_RANGE_CONNECTOR, 
oc.getRangeMap());
+            ord.getAnnotations().put(OperatorAnnotations.USE_STATIC_RANGE, 
oc.getRangeMap());
         }
         return new Pair<>(ord, null);
     }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index cfec2de..9727d46 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -36,7 +36,6 @@ import 
org.apache.asterix.algebra.base.ILangExpressionToPlanTranslatorFactory;
 import org.apache.asterix.api.http.server.ResultUtil;
 import org.apache.asterix.common.api.INodeJobTracker;
 import org.apache.asterix.common.config.CompilerProperties;
-import org.apache.asterix.common.config.OptimizationConfUtil;
 import org.apache.asterix.common.exceptions.ACIDException;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.exceptions.CompilationException;
@@ -136,7 +135,8 @@ public class APIFramework {
     private static final Set<String> CONFIGURABLE_PARAMETER_NAMES =
             ImmutableSet.of(CompilerProperties.COMPILER_JOINMEMORY_KEY, 
CompilerProperties.COMPILER_GROUPMEMORY_KEY,
                     CompilerProperties.COMPILER_SORTMEMORY_KEY, 
CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY,
-                    CompilerProperties.COMPILER_PARALLELISM_KEY, 
FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
+                    CompilerProperties.COMPILER_PARALLELISM_KEY, 
CompilerProperties.COMPILER_SORT_PARALLEL_KEY,
+                    CompilerProperties.COMPILER_SORT_SAMPLES_KEY, 
FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
                     FuzzyUtils.SIM_FUNCTION_PROP_NAME, 
FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
                     StartFeedStatement.WAIT_FOR_COMPLETION, 
FeedActivityDetails.FEED_POLICY_NAME,
                     FeedActivityDetails.COLLECT_LOCATIONS, 
SqlppQueryRewriter.INLINE_WITH_OPTION,
@@ -336,12 +336,17 @@ public class APIFramework {
         int textSearchFrameLimit = 
getFrameLimit(CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY,
                 (String) 
querySpecificConfig.get(CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY),
                 compilerProperties.getTextSearchMemorySize(), frameSize, 
MIN_FRAME_LIMIT_FOR_TEXTSEARCH, sourceLoc);
-        final PhysicalOptimizationConfig physOptConf = 
OptimizationConfUtil.getPhysicalOptimizationConfig();
+        int sortNumSamples = getSortSamples(compilerProperties, 
querySpecificConfig);
+        boolean fullParallelSort = getSortParallel(compilerProperties, 
querySpecificConfig);
+
+        final PhysicalOptimizationConfig physOptConf = new 
PhysicalOptimizationConfig();
         physOptConf.setFrameSize(frameSize);
         physOptConf.setMaxFramesExternalSort(sortFrameLimit);
         physOptConf.setMaxFramesExternalGroupBy(groupFrameLimit);
         physOptConf.setMaxFramesForJoin(joinFrameLimit);
         physOptConf.setMaxFramesForTextSearch(textSearchFrameLimit);
+        physOptConf.setSortParallel(fullParallelSort);
+        physOptConf.setSortSamples(sortNumSamples);
 
         return physOptConf;
     }
@@ -495,6 +500,25 @@ public class APIFramework {
         return parameter == null ? parallelismInConfiguration : 
integerIPropertyInterpreter.parse(parameter);
     }
 
+    private boolean getSortParallel(CompilerProperties compilerProperties, 
Map<String, Object> querySpecificConfig) {
+        String valueInQuery = (String) 
querySpecificConfig.get(CompilerProperties.COMPILER_SORT_PARALLEL_KEY);
+        if (valueInQuery != null) {
+            return OptionTypes.BOOLEAN.parse(valueInQuery);
+        }
+        return compilerProperties.getSortParallel();
+    }
+
+    private int getSortSamples(CompilerProperties compilerProperties, 
Map<String, Object> querySpecificConfig) {
+        String valueInQuery = (String) 
querySpecificConfig.get(CompilerProperties.COMPILER_SORT_SAMPLES_KEY);
+        if (valueInQuery != null) {
+            int parsedNumSamples = OptionTypes.INTEGER.parse(valueInQuery);
+            if (parsedNumSamples > 0) {
+                return parsedNumSamples;
+            }
+        }
+        return compilerProperties.getSortSamples();
+    }
+
     // Validates if the query contains unsupported query parameters.
     private static Map<String, Object> validateConfig(Map<String, Object> 
config, SourceLocation sourceLoc)
             throws AlgebricksException {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
index 1913683..c7db521 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
@@ -86,6 +86,7 @@ public class OperatorResourcesComputer {
             case INDEX_INSERT_DELETE_UPSERT:
             case INSERT_DELETE_UPSERT:
             case INTERSECT:
+            case FORWARD:
                 return getOperatorRequiredMemory(operator, frameSize);
             case LEFT_OUTER_UNNEST_MAP:
             case UNNEST_MAP:

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/PlanStagesGenerator.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/PlanStagesGenerator.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/PlanStagesGenerator.java
index 8b32375..0023a7a 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/PlanStagesGenerator.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/PlanStagesGenerator.java
@@ -38,6 +38,7 @@ import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOper
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.ForwardOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -67,13 +68,22 @@ import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.WriteResultO
 import 
org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
 import org.apache.hyracks.util.annotations.NotThreadSafe;
 
+/**
+ * Visits the operator first. Then, it visits all its inputs (pre-order 
traversal). When it visits an operator, it adds
+ * the operator to the current stage. If the operator is a multi-stage 
operator, it also adds the operator to a queue
+ * to re-visit the operator again to create the other stage.
+ */
 @NotThreadSafe
 public class PlanStagesGenerator implements ILogicalOperatorVisitor<Void, 
Void> {
 
-    private static final int JOIN_FIRST_INPUT = 1;
-    private static final int JOIN_SECOND_INPUT = 2;
+    private static final int JOIN_NON_BLOCKING_INPUT = 0;
+    private static final int JOIN_BLOCKING_INPUT = 1;
+    private static final int JOIN_NUM_INPUTS = 2;
+    private static final int FORWARD_NON_BLOCKING_INPUT = 0;
+    private static final int FORWARD_BLOCKING_INPUT = 1;
+    private static final int FORWARD_NUM_INPUTS = 2;
     private final Set<ILogicalOperator> visitedOperators = new HashSet<>();
-    private final LinkedList<ILogicalOperator> pendingBlockingOperators = new 
LinkedList<>();
+    private final LinkedList<ILogicalOperator> pendingMultiStageOperators = 
new LinkedList<>();
     private final List<PlanStage> stages = new ArrayList<>();
     private PlanStage currentStage;
     private int stageCounter;
@@ -163,7 +173,7 @@ public class PlanStagesGenerator implements 
ILogicalOperatorVisitor<Void, Void>
 
     @Override
     public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws 
AlgebricksException {
-        // Makes sure that the downstream of a replicate operator is only 
visited once.
+        // make sure that the downstream of a replicate operator is visited 
only once.
         if (!visitedOperators.contains(op)) {
             visitedOperators.add(op);
             visit(op);
@@ -175,7 +185,7 @@ public class PlanStagesGenerator implements 
ILogicalOperatorVisitor<Void, Void>
 
     @Override
     public Void visitSplitOperator(SplitOperator op, Void arg) throws 
AlgebricksException {
-        // Makes sure that the downstream of a split operator is only visited 
once.
+        // make sure that the downstream of a split operator is visited only 
once.
         if (!visitedOperators.contains(op)) {
             visitedOperators.add(op);
             visit(op);
@@ -300,59 +310,82 @@ public class PlanStagesGenerator implements 
ILogicalOperatorVisitor<Void, Void>
         return null;
     }
 
+    @Override
+    public Void visitForwardOperator(ForwardOperator op, Void arg) throws 
AlgebricksException {
+        visit(op);
+        return null;
+    }
+
     public List<PlanStage> getStages() {
         return stages;
     }
 
     private void visit(ILogicalOperator op) throws AlgebricksException {
         addToStage(op);
-        if (!pendingBlockingOperators.isEmpty()) {
-            final ILogicalOperator firstPending = 
pendingBlockingOperators.pop();
-            visitBlocking(firstPending);
+        if (!pendingMultiStageOperators.isEmpty()) {
+            final ILogicalOperator firstPending = 
pendingMultiStageOperators.pop();
+            visitMultiStageOp(firstPending);
         }
     }
 
-    private void visitBlocking(ILogicalOperator blockingOp) throws 
AlgebricksException {
+    private void visitMultiStageOp(ILogicalOperator multiStageOp) throws 
AlgebricksException {
         final PlanStage blockingOpStage = new PlanStage(++stageCounter);
-        blockingOpStage.getOperators().add(blockingOp);
+        blockingOpStage.getOperators().add(multiStageOp);
         stages.add(blockingOpStage);
         currentStage = blockingOpStage;
-        switch (blockingOp.getOperatorTag()) {
+        switch (multiStageOp.getOperatorTag()) {
             case INNERJOIN:
             case LEFTOUTERJOIN:
-                // visit only the second input
-                ILogicalOperator joinSecondInput = 
getJoinOperatorInput(blockingOp, JOIN_SECOND_INPUT);
-                joinSecondInput.accept(this, null);
+                // visit only the blocking input creating a new stage
+                ILogicalOperator newStageOperator = getInputAt(multiStageOp, 
JOIN_BLOCKING_INPUT, JOIN_NUM_INPUTS);
+                newStageOperator.accept(this, null);
                 break;
             case GROUP:
             case ORDER:
-                visitInputs(blockingOp);
+                visitInputs(multiStageOp);
+                break;
+            case FORWARD:
+                // visit only the blocking input creating a new stage
+                ILogicalOperator newStageOp = getInputAt(multiStageOp, 
FORWARD_BLOCKING_INPUT, FORWARD_NUM_INPUTS);
+                newStageOp.accept(this, null);
                 break;
             default:
-                throw new IllegalStateException("Unrecognized blocking 
operator: " + blockingOp.getOperatorTag());
+                throw new IllegalStateException("Unrecognized blocking 
operator: " + multiStageOp.getOperatorTag());
         }
     }
 
+    /**
+     * Adds the op argument to the current stage. If the operator is a 
multi-stage, it adds the operator to the pending
+     * list and continues on the branch that is non-blocking (i.e., the branch 
continuing on the same current stage)
+     * @param op to be added to the current stage
+     * @throws AlgebricksException
+     */
     private void addToStage(ILogicalOperator op) throws AlgebricksException {
         currentStage.getOperators().add(op);
         switch (op.getOperatorTag()) {
             case INNERJOIN:
             case LEFTOUTERJOIN:
-                pendingBlockingOperators.add(op);
+                pendingMultiStageOperators.add(op);
                 // continue on the same stage
-                final ILogicalOperator joinFirstInput = 
getJoinOperatorInput(op, JOIN_FIRST_INPUT);
-                joinFirstInput.accept(this, null);
+                final ILogicalOperator joinNonBlockingInput = getInputAt(op, 
JOIN_NON_BLOCKING_INPUT, JOIN_NUM_INPUTS);
+                joinNonBlockingInput.accept(this, null);
                 break;
             case GROUP:
                 if (isBlockingGroupBy((GroupByOperator) op)) {
-                    pendingBlockingOperators.add(op);
+                    pendingMultiStageOperators.add(op);
                     return;
                 }
                 // continue on the same stage
                 visitInputs(op);
                 break;
             case ORDER:
-                pendingBlockingOperators.add(op);
+                pendingMultiStageOperators.add(op);
+                break;
+            case FORWARD:
+                pendingMultiStageOperators.add(op);
+                // continue on the same current stage through the branch that 
is non-blocking
+                ILogicalOperator nonBlockingInput = getInputAt(op, 
FORWARD_NON_BLOCKING_INPUT, FORWARD_NUM_INPUTS);
+                nonBlockingInput.accept(this, null);
                 break;
             default:
                 visitInputs(op);
@@ -397,15 +430,16 @@ public class PlanStagesGenerator implements 
ILogicalOperatorVisitor<Void, Void>
         return false;
     }
 
-    private ILogicalOperator getJoinOperatorInput(ILogicalOperator op, int 
inputNum) {
-        if (inputNum != JOIN_FIRST_INPUT && inputNum != JOIN_SECOND_INPUT) {
-            throw new IllegalArgumentException("invalid input number for join 
operator");
-        }
+    private ILogicalOperator getInputAt(ILogicalOperator op, int inputIndex, 
int numInputs) {
         final List<Mutable<ILogicalOperator>> inputs = op.getInputs();
-        if (inputs.size() != 2) {
-            throw new IllegalStateException("Join must have exactly two 
inputs. Current inputs: " + inputs.size());
+        int inSize = inputs.size();
+        if (inSize != numInputs) {
+            throw new IllegalStateException("Op must have exactly " + 
numInputs + " inputs. Current inputs: " + inSize);
+        }
+        if (inputIndex >= inSize) {
+            throw new IllegalArgumentException("invalid input index for 
operator");
         }
-        return op.getInputs().get(inputNum - 1).getValue();
+        return inputs.get(inputIndex).getValue();
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/RequiredCapacityVisitor.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/RequiredCapacityVisitor.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/RequiredCapacityVisitor.java
index 89fbcb2..368a244 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/RequiredCapacityVisitor.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/RequiredCapacityVisitor.java
@@ -36,6 +36,7 @@ import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOper
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.ForwardOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator;
 import 
org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -304,6 +305,12 @@ public class RequiredCapacityVisitor implements 
ILogicalOperatorVisitor<Void, Vo
         return null;
     }
 
+    @Override
+    public Void visitForwardOperator(ForwardOperator op, Void arg) throws 
AlgebricksException {
+        visitInternal(op, true);
+        return null;
+    }
+
     // Calculates the memory usage for exchange operators.
     private void calculateMemoryUsageForExchange(ExchangeOperator op) throws 
AlgebricksException {
         visitInternal(op, false);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
index 5f76568..d0adcda 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
@@ -23,7 +23,6 @@ import java.io.DataOutputStream;
 import java.util.HashMap;
 import java.util.Map;
 
-import 
org.apache.asterix.external.library.java.base.ByteArrayAccessibleInputStream;
 import org.apache.asterix.external.parser.JSONDataParser;
 import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
 import org.apache.asterix.om.base.IAObject;
@@ -35,6 +34,7 @@ import org.apache.asterix.translator.ResultProperties;
 import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.result.IResultSet;
+import org.apache.hyracks.data.std.util.ByteArrayAccessibleInputStream;
 import org.apache.hyracks.data.std.util.ByteArrayAccessibleOutputStream;
 
 import com.fasterxml.jackson.databind.JsonNode;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/main/resources/cc.conf
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf 
b/asterixdb/asterix-app/src/main/resources/cc.conf
index 6971b2b..8877be8 100644
--- a/asterixdb/asterix-app/src/main/resources/cc.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc.conf
@@ -53,6 +53,7 @@ compiler.sortmemory=320KB
 compiler.groupmemory=160KB
 compiler.joinmemory=256KB
 compiler.textsearchmemory=160KB
+compiler.sort.parallel=false
 messaging.frame.size=4096
 messaging.frame.count=512
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPartitionComputerFactory.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPartitionComputerFactory.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPartitionComputerFactory.java
index 385f6a2..66b01a7 100644
--- 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPartitionComputerFactory.java
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPartitionComputerFactory.java
@@ -23,6 +23,7 @@ import java.util.Iterator;
 import java.util.List;
 
 import org.apache.hyracks.api.comm.IFrameTupleAccessor;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
 import org.apache.hyracks.api.dataflow.value.ITuplePartitionComputer;
 import org.apache.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -41,7 +42,7 @@ public class TestPartitionComputerFactory implements 
ITuplePartitionComputerFact
     }
 
     @Override
-    public ITuplePartitionComputer createPartitioner() {
+    public ITuplePartitionComputer createPartitioner(IHyracksTaskContext ctx) {
         return new ITuplePartitionComputer() {
             private final List<Integer> destinations =
                     new 
ArrayList<Integer>(TestPartitionComputerFactory.this.destinations);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
index debb3f9..6e0413c 100644
--- 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
@@ -60,6 +60,11 @@ import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
 
+/**
+ * Runs the optimization tests. The current configuration runs the tests with 
parallel sort disabled.
+ * Note: when adding a new test case and it includes sorting, provide another 
test case and enable parallel sort in the
+ * query by setting the property (append the test case name with "_ps")
+ */
 @RunWith(Parameterized.class)
 public class OptimizerTest {
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01_ps.sqlpp
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01_ps.sqlpp
 
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01_ps.sqlpp
new file mode 100644
index 0000000..4d8c4ff
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01_ps.sqlpp
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Test that left-outer-join may use two available indexes, one 
for primary index in prob subtree and another for secondary btree index in 
index subtree.
+ * Issue        : 730, 741
+ * Expected Res : Success
+ * Date         : 8th May 2014
+ */
+
+drop  dataverse test if exists;
+create  dataverse test;
+
+use test;
+
+
+create type test.TwitterUserType as
+ closed {
+  `screen-name` : string,
+  lang : string,
+  `friends-count` : integer,
+  `statuses-count` : integer,
+  name : string,
+  `followers-count` : integer
+};
+
+create type test.TweetMessageType as
+ closed {
+  tweetid : bigint,
+  user : TwitterUserType,
+  `sender-location` : point,
+  `send-time` : datetime,
+  `referred-topics` : {{string}},
+  `message-text` : string,
+  countA : integer,
+  countB : integer
+};
+
+create  dataset TweetMessages(TweetMessageType) primary key tweetid;
+
+create  index twmSndLocIx  on TweetMessages (`sender-location`) type rtree;
+
+create  index msgCountAIx  on TweetMessages (countA) type btree;
+
+create  index msgCountBIx  on TweetMessages (countB) type btree;
+
+create  index msgTextIx  on TweetMessages (`message-text`) type keyword;
+
+write output to 
asterix_nc1:"rttest/btree-index-join_leftouterjoin-probe-pidx-with-join-btree-sidx_01.adm";
+set `compiler.sort.parallel` "true";
+
+select element {'tweetid1':t1.tweetid,'count1':t1.countA,'t2info':(
+        select element {'tweetid2':t2.tweetid,'count2':t2.countB}
+        from  TweetMessages as t2
+        where (t1.countA /*+ indexnl */  = t2.countB)
+        order by t2.tweetid
+    )}
+from  TweetMessages as t1
+where (t1.tweetid < test.bigint('10'))
+order by t1.tweetid
+;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_02_ps.sqlpp
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_02_ps.sqlpp
 
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_02_ps.sqlpp
new file mode 100644
index 0000000..a5c5fe4
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_02_ps.sqlpp
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Test that left-outer-join may use two available indexes, one 
for primary index in prob subtree and
+ *                another for secondary btree index in index subtree. p_sort 
enabled.
+ * Issue        : 730, 741
+ * Expected Res : Success
+ * Date         : 8th May 2014
+ */
+
+drop  dataverse test if exists;
+create  dataverse test;
+
+use test;
+
+
+create type test.TwitterUserType as
+ closed {
+  `screen-name` : string,
+  lang : string,
+  `friends-count` : integer,
+  `statuses-count` : integer,
+  name : string,
+  `followers-count` : integer
+};
+
+create type test.TweetMessageType as
+ closed {
+  tweetid : bigint,
+  user : TwitterUserType,
+  `sender-location` : point,
+  `send-time` : datetime,
+  `referred-topics` : {{string}},
+  `message-text` : string,
+  countA : integer,
+  countB : integer
+};
+
+create  dataset TweetMessages(TweetMessageType) primary key tweetid;
+
+create  index twmSndLocIx  on TweetMessages (`sender-location`) type rtree;
+
+create  index msgCountAIx  on TweetMessages (countA) type btree;
+
+create  index msgCountBIx  on TweetMessages (countB) type btree;
+
+create  index msgTextIx  on TweetMessages (`message-text`) type keyword;
+
+write output to 
asterix_nc1:"rttest/btree-index-join_leftouterjoin-probe-pidx-with-join-btree-sidx_02.adm";
+
+set `compiler.sort.parallel` "true";
+
+select element {'tweetid1':t1.tweetid,'count1':t1.countA,'t2info':(
+        select element {'tweetid2':t2.tweetid,'count2':t2.countB}
+        from  TweetMessages as t2
+        where ((t1.countA /*+ indexnl */  = t2.countB) and (t1.tweetid != 
t2.tweetid))
+        order by t2.tweetid
+    )}
+from  TweetMessages as t1
+where (t1.tweetid < test.bigint('10'))
+order by t1.tweetid
+;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-68_ps.sqlpp
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-68_ps.sqlpp
 
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-68_ps.sqlpp
new file mode 100644
index 0000000..520a3f0
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-68_ps.sqlpp
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ *  Description     : Secondary BTree Index intersection with between operator 
(ASTERIXDB-2448). p_sort enabled.
+ *  Expected Result : Success
+ */
+
+drop  dataverse tpch if exists;
+create  dataverse tpch;
+
+use tpch;
+
+create type tpch.OrderType as
+ closed {
+  o_orderkey : bigint,
+  o_custkey : bigint,
+  o_orderstatus : string,
+  o_totalprice : double,
+  o_orderdate : string,
+  o_orderpriority : string,
+  o_clerk : string,
+  o_shippriority : bigint,
+  o_comment : string
+};
+
+create  dataset Orders(OrderType) primary key o_orderkey;
+
+create index idx_custkey on Orders (o_custkey) type btree;
+
+create index idx_orderpriority on Orders (o_orderpriority) type btree;
+
+set `compiler.sort.parallel` "true";
+
+select o_custkey, o_orderkey, o_orderstatus from Orders
+where
+  o_orderpriority = '1-URGENT' and
+  o_custkey between 40 and 43
+order by o_custkey, o_orderkey;
+

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-01-disable-idxonly_ps.sqlpp
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-01-disable-idxonly_ps.sqlpp
 
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-01-disable-idxonly_ps.sqlpp
new file mode 100644
index 0000000..b986780
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-01-disable-idxonly_ps.sqlpp
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ *  Description     : Secondary BTree Index index-only selection plan 
verification test
+ *                  : The test is intended to verify that the secondary BTree 
index is used in the optimized query plan.
+ *                  : In this plan, we fetch PK and SK based on a select 
condition that utilizes a secondary index.
+ *                  : The plan should have two paths after the secondary 
index-lookup.
+ *                  : The left path:
+ *                      ... -> unnest-map (sidx) -> split -> unnest-map (pidx) 
-> select -> union -> ...
+ *                  : The right path:
+ *                      ... -> unnest-map (sidx) -> split ->                   
          -> union -> ...
+ *                  : However, we set the "noindexonly" option to true. So, 
the index-only plan should not be triggered.
+ *  Expected Result : Success
+ *
+*/
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type MyRecord as closed {
+  id: int64,
+  docid: int64,
+  val1: int64,
+  title: string,
+  point: point,
+  kwds: string,
+  line1: line,
+  line2: line,
+  poly1: polygon,
+  poly2: polygon,
+  rec: rectangle,
+  circle: circle
+};
+
+create dataset MyData(MyRecord) primary key id;
+
+create index btree_index_docid on MyData(docid) type btree;
+create index btree_index_val1 on MyData(val1) type btree;
+create index rtree_index_point on MyData(point) type rtree;
+create index rtree_index_rec on MyData(rec) type rtree;
+create index ngram_index_title on MyData(title) type ngram(3);
+create index keyword_index_title on MyData(title) type keyword;
+
+set noindexonly 'true';
+set `compiler.sort.parallel` "true";
+
+select element {"pk":o.id, "sk":o.docid}
+from MyData o
+where o.docid < 3
+order by o.id;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-01_ps.sqlpp
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-01_ps.sqlpp
 
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-01_ps.sqlpp
new file mode 100644
index 0000000..e5c4e0c
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-01_ps.sqlpp
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ *  Description     : Secondary BTree Index index-only selection plan 
verification test
+ *                  : The test is intended to verify that the secondary BTree 
index is used in the optimized query plan.
+ *                  : In this plan, we fetch PK and SK based on a select 
condition that utilizes a secondary index.
+ *                  : The plan should have two paths after the secondary 
index-lookup.
+ *                  : The left path:
+ *                      ... -> unnest-map (sidx) -> split -> unnest-map (pidx) 
-> select -> union -> ...
+ *                  : The right path:
+ *                      ... -> unnest-map (sidx) -> split ->                   
          -> union -> ...
+ *  Expected Result : Success
+ *
+*/
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type MyRecord as closed {
+  id: int64,
+  docid: int64,
+  val1: int64,
+  title: string,
+  point: point,
+  kwds: string,
+  line1: line,
+  line2: line,
+  poly1: polygon,
+  poly2: polygon,
+  rec: rectangle,
+  circle: circle
+};
+
+create dataset MyData(MyRecord) primary key id;
+
+create index btree_index_docid on MyData(docid) type btree;
+create index btree_index_val1 on MyData(val1) type btree;
+create index rtree_index_point on MyData(point) type rtree;
+create index rtree_index_rec on MyData(rec) type rtree;
+create index ngram_index_title on MyData(title) type ngram(3);
+create index keyword_index_title on MyData(title) type keyword;
+
+set `compiler.sort.parallel` "true";
+
+select element {"pk":o.id, "sk":o.docid}
+from MyData o
+where o.docid < 3
+order by o.id;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/test/resources/optimizerts/queries/introhashpartitionmerge.sqlpp
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/introhashpartitionmerge.sqlpp
 
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/introhashpartitionmerge.sqlpp
index 103aae8..9921e58 100644
--- 
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/introhashpartitionmerge.sqlpp
+++ 
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/introhashpartitionmerge.sqlpp
@@ -36,6 +36,9 @@ create  nodegroup group1 if not exists  on
 create  dataset TOKENSRANKEDADM(TOKENSRANKEDADMType) primary key rank on 
group1;
 
 write output to asterix_nc1:"rttest/introhashpartitionmerge.adm";
+
+SET `compiler.sort.parallel` "false";
+
 select element token2.rank
 from  TOKENSRANKEDADM as token1,
       (
@@ -44,5 +47,4 @@ from  TOKENSRANKEDADM as token1,
     order by tokenRanked.rank
 ) as token2
 where (token1.token = token2.token)
-order by token2.rank
-;
+order by token2.rank;

Reply via email to