This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 7481bc8d4f1bb6f69b3c4356f1be98bbc82aeef7 Merge: bfea25b 8b5b5ce Author: Dmitry Lychagin <[email protected]> AuthorDate: Thu Mar 25 18:20:30 2021 -0700 Merge branch 'gerrit/cheshire-cat' Change-Id: Ia23c75c9cc5852837276eed134876690ce3e3901 .../asterix/optimizer/base/RuleCollections.java | 9 +- .../IntroduceSecondaryIndexInsertDeleteRule.java | 3 +- .../PushAggFuncIntoStandaloneAggregateRule.java | 142 ++++++----- .../rules/PushAggregateIntoNestedSubplanRule.java | 65 ++--- .../rules/RemoveRedundantListifyRule.java | 100 +++++--- .../rules/SetAsterixPhysicalOperatorsRule.java | 2 +- .../optimizer/rules/am/RTreeAccessMethod.java | 2 +- ...InlineSubplanInputForNestedTupleSourceRule.java | 5 + .../rules/subplan/SubplanFlatteningUtil.java | 12 +- .../temporal/TranslateIntervalExpressionRule.java | 115 +++++---- .../optimizer/rules/util/IntervalJoinUtils.java | 14 +- .../translator/LangExpressionToPlanTranslator.java | 4 +- asterixdb/asterix-app/pom.xml | 2 +- .../api/http/server/AbstractNCUdfServlet.java | 206 +++++++++++++-- .../asterix/api/http/server/BasicAuthServlet.java | 13 +- .../asterix/api/http/server/NCUdfApiServlet.java | 193 ++++++++------- .../api/http/server/NCUdfRecoveryServlet.java | 5 +- .../api/http/server/QueryServiceServlet.java | 74 +++--- .../api/http/server/RebalanceApiServlet.java | 26 +- .../apache/asterix/app/nc/NCAppRuntimeContext.java | 12 +- .../asterix/hyracks/bootstrap/NCApplication.java | 6 +- .../asterix-app/src/main/resources/entrypoint.py | 111 ++++++--- .../asterix/app/external/ExternalUDFLibrarian.java | 46 +++- .../app/external/IExternalUDFLibrarian.java | 7 +- .../asterix/test/common/ResultExtractor.java | 12 +- .../apache/asterix/test/common/TestExecutor.java | 162 +++++++++++- .../test/dataflow/LSMFlushRecoveryTest.java | 2 +- .../TweetSent/{sentiment.py => crashy.py} | 19 +- .../src/test/resources/TweetSent/roundtrip.py | 4 + .../src/test/resources/TweetSent/sentiment.py | 6 +- asterixdb/asterix-app/src/test/resources/cc.conf | 1 + .../agg_filter_01/agg_filter_01.10.sqlpp | 58 +++++ .../agg_filter_01/agg_filter_01.9.sqlpp | 55 +++++ .../queries/subquery/query-ASTERIXDB-2845.sqlpp} | 45 +++- .../agg_filter_01/agg_filter_01.10.plan | 27 ++ .../agg_filter_01/agg_filter_01.3.plan | 28 +-- .../agg_filter_01/agg_filter_01.4.plan | 41 ++- .../agg_filter_01/agg_filter_01.5.plan | 94 +++---- .../agg_filter_01/agg_filter_01.9.plan | 16 ++ .../results/q01_pricing_summary_report_nt_ps.plan | 8 +- .../optimizerts/results/query-ASTERIXDB-1806.plan | 4 +- .../results/query-ASTERIXDB-1806_ps.plan | 8 +- .../optimizerts/results/subquery/exists.plan | 12 +- .../optimizerts/results/subquery/exists_ps.plan | 24 +- .../optimizerts/results/subquery/not_exists.plan | 12 +- .../results/subquery/not_exists_ps.plan | 24 +- .../results/subquery/query-ASTERIXDB-2845.plan | 123 +++++++++ .../optimizerts/results/tpch/q12_shipping.plan | 10 +- .../results/tpch/q12_shipping_broadcast.plan | 8 +- .../results/tpch/q12_shipping_broadcast_ps.plan | 16 +- .../optimizerts/results/tpch/q12_shipping_ps.plan | 20 +- .../agg_filter_01/agg_filter_01.10.query.sqlpp} | 23 +- .../agg_filter_01/agg_filter_01.9.query.sqlpp} | 20 +- .../bad-ext-function-ddl-1.2.lib.sqlpp | 2 +- .../crash.0.ddl.sqlpp} | 4 +- .../crash.1.lib.sqlpp} | 3 +- .../crash.2.ddl.sqlpp} | 4 +- .../crash.3.query.sqlpp} | 6 +- .../create-or-replace-function-1.2.lib.sqlpp | 2 +- .../deterministic/deterministic.1.lib.sqlpp | 2 +- .../exception_create_system_library.1.lib.sqlpp | 2 +- .../getCapital/getCapital.1.lib.sqlpp | 2 +- .../getCapital_open/getCapital_open.1.lib.sqlpp | 2 +- .../library_list_api_multipart.1.post.http} | 8 +- .../library_list_api_multipart.2.post.http} | 9 +- .../library_list_api_multipart.3.post.http} | 7 +- .../library_list_api_multipart.4.post.http} | 8 +- .../library_list_api_multipart.5.post.http} | 7 +- .../keyword_detector/keyword_detector.1.lib.sqlpp | 2 +- .../library_list_api.0.ddl.sqlpp} | 4 +- .../library_list_api.1.post.http} | 8 +- .../library_list_api.2.get.http} | 3 +- .../library_list_api_multipart.0.ddl.sqlpp} | 9 +- .../library_list_api_multipart.1.post.http} | 9 +- .../library_list_api_multipart.2.post.http} | 8 +- .../library_list_api_multipart.3.post.http} | 10 +- .../library_list_api_multipart.4.post.http} | 10 +- .../library_list_api_multipart.5.get.http} | 3 +- .../my_array_sum/my_array_sum.1.lib.sqlpp | 2 +- .../mysentiment/mysentiment.1.lib.sqlpp | 2 +- .../mysentiment.6.query.sqlpp} | 6 +- ...ntiment.6.ddl.sqlpp => mysentiment.7.ddl.sqlpp} | 0 .../mysentiment_multipart.0.ddl.sqlpp} | 4 +- .../mysentiment_multipart.1.lib.sqlpp} | 3 +- .../mysentiment_multipart.2.ddl.sqlpp} | 5 +- .../mysentiment_multipart.3.query.sqlpp} | 4 +- .../mysentiment_multipart.4.query.sqlpp} | 4 +- .../mysentiment_multipart.5.query.sqlpp} | 4 +- .../mysentiment_multipart.6.ddl.sqlpp} | 2 +- .../mysentiment_twitter.0.ddl.sqlpp} | 12 +- .../mysentiment_twitter.1.update.sqlpp} | 5 +- .../mysentiment_twitter.10.query.sqlpp} | 5 +- .../mysentiment_twitter.11.query.sqlpp} | 8 +- .../mysentiment_twitter.12.query.sqlpp} | 6 +- .../mysentiment_twitter.13.ddl.sqlpp} | 6 +- .../mysentiment_twitter.14.query.sqlpp} | 6 +- .../mysentiment_twitter.15.query.sqlpp} | 8 +- .../mysentiment_twitter.16.ddl.sqlpp} | 6 +- .../mysentiment_twitter.17.query.sqlpp} | 6 +- .../mysentiment_twitter.18.query.sqlpp} | 8 +- .../mysentiment_twitter.19.ddl.sqlpp} | 6 +- .../mysentiment_twitter.2.lib.sqlpp} | 3 +- .../mysentiment_twitter.20.query.sqlpp} | 8 +- .../mysentiment_twitter.21.ddl.sqlpp} | 0 .../mysentiment_twitter.3.ddl.sqlpp} | 5 +- .../mysentiment_twitter.4.query.sqlpp} | 5 +- .../mysentiment_twitter.5.query.sqlpp} | 8 +- .../mysentiment_twitter.6.query.sqlpp} | 8 +- .../mysentiment_twitter.7.query.sqlpp} | 10 +- .../mysentiment_twitter.8.update.sqlpp} | 26 +- .../mysentiment_twitter.9.query.sqlpp} | 5 +- .../external-library/mysum/mysum.1.lib.sqlpp | 2 +- .../mysum_bad_credential.1.lib.sqlpp | 2 +- .../mysum_bad_credential.2.lib.sqlpp | 2 +- .../mysum_bad_credential.3.lib.sqlpp | 2 +- .../mysum_dropinuse/mysum_dropinuse.1.lib.sqlpp | 2 +- .../py_function_error.1.lib.sqlpp | 2 +- .../py_function_error.2.ddl.sqlpp | 3 + .../py_function_error.4.query.sqlpp} | 10 +- .../py_nested_access/py_nested_access.1.lib.sqlpp | 2 +- .../py_nested_access.10.query.sqlpp | 2 +- .../py_nested_access.11.query.sqlpp | 2 +- .../py_nested_access.12.query.sqlpp | 2 +- .../py_nested_access.13.query.sqlpp | 2 +- .../py_nested_access.4.query.sqlpp | 5 +- .../py_nested_access.5.query.sqlpp | 6 +- .../py_nested_access.6.query.sqlpp | 2 +- .../py_nested_access.7.query.sqlpp | 2 +- .../py_nested_access.8.query.sqlpp | 2 +- .../py_nested_access.9.query.sqlpp | 2 +- .../python-fn-escape/python-fn-escape.1.lib.sqlpp | 2 +- .../type_validation.0.ddl.sqlpp} | 4 +- .../type_validation.1.lib.sqlpp} | 2 +- .../type_validation.2.ddl.sqlpp} | 6 +- .../type_validation.3.query.sqlpp} | 5 +- .../type_validation.4.ddl.sqlpp} | 1 + .../return_invalid_type.1.lib.sqlpp | 2 +- .../mysentiment.0.ddl.sqlpp} | 4 +- .../mysentiment.1.lib.sqlpp | 2 +- .../mysentiment.2.ddl.sqlpp} | 4 +- .../mysentiment.3.query.sqlpp} | 5 +- .../mysentiment.4.ddl.sqlpp} | 0 .../type_validation/type_validation.1.lib.sqlpp | 2 +- .../udf_metadata/udf_metadata.1.lib.sqlpp | 4 +- .../upperCase/upperCase.1.lib.sqlpp | 2 +- .../exception_create_system_adapter.1.lib.sqlpp | 2 +- ...feed-with-external-adapter-cross-dv.1.lib.sqlpp | 2 +- .../feed-with-external-adapter.1.lib.sqlpp | 2 +- .../feed-with-external-function.1.lib.sqlpp | 2 +- .../all_datasets/all_datasets.10.post.http | 5 +- .../all_datasets/all_datasets.4.post.http | 4 +- .../all_datasets_compressed.10.post.http | 5 +- .../all_datasets_compressed.4.post.http | 4 +- .../duplicate_location.3.post.http | 8 +- .../empty_location/empty_location.3.post.http | 5 +- .../identical_location.3.post.http | 8 +- .../rebalance/metadata/metadata.1.post.http | 6 +- .../miss_dataverse/miss_dataverse.3.post.http | 5 +- .../nonexist_dataset/nonexist_dataset.1.post.http | 6 +- .../single_dataset/single_dataset.4.post.http | 6 +- .../single_dataset/single_dataset.8.post.http | 7 +- .../single_dataset_compressed.4.post.http | 6 +- .../single_dataset_compressed.8.post.http | 7 +- .../single_dataset_with_index.4.post.http | 6 +- .../single_dataset_with_index.9.post.http | 7 +- ...ingle_dataset_with_index_compressed.4.post.http | 6 +- ...ingle_dataset_with_index_compressed.9.post.http | 7 +- .../single_dataverse/single_dataverse.10.post.http | 6 +- .../single_dataverse/single_dataverse.4.post.http | 5 +- .../single_dataverse_compressed.10.post.http | 6 +- .../single_dataverse_compressed.4.post.http | 5 +- .../replication/bulkload/bulkload.10.post.http | 5 +- .../replication/bulkload/bulkload.9.post.http | 5 +- .../bulkload.10.post.http | 5 +- .../bulkload_with_compression/bulkload.9.post.http | 5 +- .../flushed_component.6.post.http | 5 +- .../flushed_component.7.post.http | 4 +- .../flushed_component_compressed.10.post.http | 5 +- .../flushed_component_compressed.11.post.http | 5 +- .../mem_component_recovery.10.post.http | 5 +- .../mem_component_recovery.9.post.http | 5 +- .../metadata_failover.6.post.http | 5 +- .../metadata_failover.7.post.http | 4 +- .../release_partition.3.post.http | 4 +- .../query-ASTERIXDB-2845.1.ddl.sqlpp} | 17 +- .../query-ASTERIXDB-2845.2.update.sqlpp} | 26 +- .../query-ASTERIXDB-2845.3.query.sqlpp} | 40 +-- .../src/test/resources/runtimets/rebalance.xml | 2 +- .../agg_filter_01/agg_filter_01.10.adm | 10 + .../agg_filter_01/agg_filter_01.9.adm | 1 + .../results/api/feed-stats/feed-stats.1.adm | 10 - .../results/api/feed-stats/feed-stats.5.regexjson | 13 + .../invalid-library-params.1.regexjson | 3 + .../results/external-library/crash/crash.1.adm | 1 + .../library_list_api/library_list_ap1.1.regexjson | 1 + .../library_list_api/library_list_ap1.2.regexjson | 5 + .../library_list_api.1.regexjson | 1 + .../library_list_api.2.regexjson | 1 + .../library_list_api.3.regexjson | 1 + .../library_list_api.4.regexjson | 1 + .../library_list_api.5.regexjson | 20 ++ .../external-library/mysentiment/mysentiment.4.adm | 1 + .../mysentiment_twitter/mysentiment_twitter.1.adm | 1 + .../mysentiment_twitter/mysentiment_twitter.10.adm | 1 + .../mysentiment_twitter/mysentiment_twitter.11.adm | 1 + .../mysentiment_twitter/mysentiment_twitter.12.adm | 1 + .../mysentiment_twitter/mysentiment_twitter.13.adm | 100 ++++++++ .../mysentiment_twitter/mysentiment_twitter.2.adm | 1 + .../mysentiment_twitter/mysentiment_twitter.3.adm | 100 ++++++++ .../mysentiment_twitter/mysentiment_twitter.4.adm | 100 ++++++++ .../mysentiment_twitter/mysentiment_twitter.5.adm | 1 + .../mysentiment_twitter/mysentiment_twitter.6.adm | 1 + .../mysentiment_twitter/mysentiment_twitter.7.adm | 1 + .../mysentiment_twitter/mysentiment_twitter.8.adm | 1 + .../mysentiment_twitter/mysentiment_twitter.9.adm | 1 + .../py_function_error/py_function_error.2.adm | 4 + .../type_validation.1.adm | 1 + .../external-library/toplevel_fn/toplevel_fn.1.adm | 1 + .../push-limit-to-primary-scan.8.adm | 24 +- .../query-ASTERIXDB-2845.3.adm | 17 ++ .../agg_filter_01/agg_filter_1.09.ast | 125 ++++++++++ .../agg_filter_01/agg_filter_1.10.ast | 137 ++++++++++ .../resources/runtimets/testsuite_it_python.xml | 37 ++- .../resources/runtimets/testsuite_it_sqlpp.xml | 20 ++ .../test/resources/runtimets/testsuite_sqlpp.xml | 5 + .../asterix/common/config/StorageProperties.java | 26 +- .../asterix/common/library/ILibraryManager.java | 6 + .../asterix/common/library/LibraryDescriptor.java | 8 +- .../asterix/common/metadata/DataverseName.java | 5 +- .../external/ipc/ExternalFunctionResultRouter.java | 43 ++-- .../src/main/resources/asx_errormsg/en.properties | 2 +- .../dataflow/FeedRecordDataFlowController.java | 10 +- .../asterix/external/ipc/PythonIPCProto.java | 113 ++++++--- .../asterix/external/ipc/PythonMessageBuilder.java | 76 ++++-- .../external/library/ExternalLibraryManager.java | 76 +++++- .../ExternalScalarPythonFunctionEvaluator.java | 274 ++++---------------- .../external/library/PythonLibraryEvaluator.java | 216 ++++++++++++++++ .../library/PythonLibraryEvaluatorFactory.java | 96 +++++++ .../external/library/PythonLibraryEvaluatorId.java | 63 +++++ .../library/msgpack/MessagePackerFromADM.java | 85 ++++--- .../library/msgpack/MessageUnpackerToADM.java | 59 ++--- .../ExternalAssignBatchRuntimeFactory.java | 273 +++++++++++++++++++- .../LibraryDeployPrepareOperatorDescriptor.java | 8 - .../external/util/ExternalDataConstants.java | 4 +- .../external/util/ExternalLibraryUtils.java | 18 +- .../lang/common/util/DataverseNameUtils.java | 85 ------- .../functions/ExternalFunctionCompilerUtil.java | 4 +- .../functions/ExternalScalarFunctionInfo.java | 5 +- .../asterix/om/functions/ExternalFunctionInfo.java | 11 +- .../om/functions/IExternalFunctionInfo.java | 2 + .../resync_failed_replica.11.post.http | 5 +- .../resync_failed_replica.12.post.http | 4 +- .../src/main/resources/Catalog.xsd | 2 + .../common/exceptions/AlgebricksException.java | 15 +- .../logical/visitors/IsomorphismUtilities.java | 8 +- .../IsomorphismVariableMappingVisitor.java | 10 +- ...lExpressionDeepCopyWithNewVariablesVisitor.java | 4 +- .../logical/visitors/VariableUtilities.java | 2 +- .../core/algebra/plan/PlanStructureVerifier.java | 7 + .../rules/EnforceStructuralPropertiesRule.java | 2 +- .../rules/InlineAssignIntoAggregateRule.java | 2 +- .../subplan/EliminateIsomorphicSubplanRule.java | 275 ++++++++++++++++----- .../hyracks/api/exceptions/HyracksException.java | 12 +- .../api/exceptions/IFormattedException.java | 19 +- .../apache/hyracks/api/util/ErrorMessageUtil.java | 21 ++ .../control/common/controllers/NCConfig.java | 5 +- .../hyracks/http/server/AbstractServlet.java | 9 +- .../storage/am/lsm/common/api/ILSMIOOperation.java | 4 + .../api/ILSMIOOperationSchedulerFactory.java | 3 +- .../impls/AbstractAsynchronousScheduler.java | 96 +++++-- .../am/lsm/common/impls/AbstractIoOperation.java | 5 + .../am/lsm/common/impls/AsynchronousScheduler.java | 37 ++- .../am/lsm/common/impls/GreedyScheduler.java | 118 ++++++--- .../am/lsm/common/impls/IoOperationExecutor.java | 29 +-- .../am/lsm/common/impls/NoOpIoOperation.java | 5 + .../am/lsm/common/impls/TracedIOOperation.java | 5 + .../lsm/btree/LSMBTreeComponentLifecycleTest.java | 3 +- .../storage/am/lsm/btree/perf/LSMTreeRunner.java | 2 +- .../am/lsm/common/test/GreedySchedulerTest.java | 133 ---------- .../am/lsm/common/test/IoSchedulerTest.java | 267 ++++++++++++++++++++ .../org/apache/hyracks/util/ThrowingConsumer.java | 14 ++ 281 files changed, 4471 insertions(+), 1740 deletions(-)
