This is an automated email from the ASF dual-hosted git repository. alsuliman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 21cf9cbe1f228cbd7d161dbd0b23b55403253dd0 Merge: b0be3fc ce5f3aa Author: Ali Alsuliman <[email protected]> AuthorDate: Wed Apr 7 11:52:34 2021 +0300 Merge branch 'cheshire-cat' into 'master' Change-Id: Ic9ebf963437bf2cac2fb48b43b41728f515133c6 .../org/apache/asterix/active/ActiveManager.java | 21 +- .../org/apache/asterix/active/IActiveRuntime.java | 5 + .../active/message/ActiveManagerMessage.java | 14 +- .../active/message/ActiveStatsRequestMessage.java | 6 +- .../active/message/StopRuntimeParameters.java | 12 +- .../operators/physical/InvertedIndexPOperator.java | 4 +- .../optimizer/rules/ConstantFoldingRule.java | 3 +- .../rules/MetaFunctionToMetaVariableRule.java | 4 +- .../rules/SetAsterixPhysicalOperatorsRule.java | 5 +- .../rules/SweepIllegalNonfunctionalFunctions.java | 3 - .../subplan/InlineAllNtsInSubplanVisitor.java | 84 +- ...InlineSubplanInputForNestedTupleSourceRule.java | 48 +- .../rules/subplan/SubplanFlatteningUtil.java | 51 +- .../rules/util/EquivalenceClassUtils.java | 84 +- .../asterix/translator/AbstractLangTranslator.java | 21 +- .../translator/LangExpressionToPlanTranslator.java | 22 +- .../apache/asterix/api/common/APIFramework.java | 7 +- .../api/http/server/ConnectorApiServlet.java | 7 +- .../asterix/api/http/server/NCUdfApiServlet.java | 4 +- .../api/http/server/QueryServiceServlet.java | 14 +- .../api/http/server/RebalanceApiServlet.java | 5 +- .../app/active/ActiveEntityEventsListener.java | 2 +- .../asterix/app/function/DatasetRewriter.java | 6 +- .../apache/asterix/app/function/FeedRewriter.java | 5 +- .../app/message/StorageCleanupRequestMessage.java | 109 + .../apache/asterix/app/message/VoidResponse.java | 63 + .../apache/asterix/app/nc/NCAppRuntimeContext.java | 22 +- .../org/apache/asterix/app/nc/RecoveryManager.java | 19 +- .../asterix/app/translator/QueryTranslator.java | 43 +- .../hyracks/bootstrap/GlobalRecoveryManager.java | 31 + .../test/dataflow/GlobalStorageCleanupTest.java | 68 + .../test/dataflow/LSMFlushRecoveryTest.java | 12 +- .../src/test/resources/TweetSent/roundtrip.py | 4 + asterixdb/asterix-app/src/test/resources/cc.conf | 1 + .../leftouterjoin/query-ASTERIXDB-2857.sqlpp | 63 + .../queries/subquery/in_let_8.sqlpp} | 43 +- .../queries/subquery/query-ASTERIXDB-2845.sqlpp | 2 +- .../optimizerts/results/ASTERIXDB-2402.plan | 28 +- .../leftouterjoin/query-ASTERIXDB-2857.plan | 39 + .../optimizerts/results/query-issue562_ps.plan | 58 +- .../optimizerts/results/subquery/exists.plan | 2 +- .../optimizerts/results/subquery/exists_ps.plan | 4 +- .../optimizerts/results/subquery/in_let_3.plan | 4 +- .../optimizerts/results/subquery/in_let_4.plan | 4 +- .../optimizerts/results/subquery/in_let_5.plan | 4 +- .../subquery/{in_let_3.plan => in_let_8.plan} | 60 +- .../optimizerts/results/subquery/not_exists.plan | 2 +- .../results/subquery/not_exists_ps.plan | 4 +- .../results/subquery/query-ASTERIXDB-2845.plan | 145 +- .../tpcds/query-ASTERIXDB-1581-correlated-2.plan | 399 +- .../tpcds/query-ASTERIXDB-1581-correlated.plan | 407 +- .../external_dataset.000.ddl.sqlpp | 66 + .../external_dataset.001.query.sqlpp} | 7 +- .../external_dataset.002.query.sqlpp} | 7 +- .../external_dataset.003.query.sqlpp} | 7 +- .../external_dataset.004.query.sqlpp} | 7 +- .../external_dataset.005.query.sqlpp} | 7 +- .../external_dataset.006.query.sqlpp} | 7 +- .../external_dataset.999.ddl.sqlpp} | 6 +- .../py_function_error.2.ddl.sqlpp | 5 +- ...2.ddl.sqlpp => py_function_error.5.query.sqlpp} | 12 +- ...2.ddl.sqlpp => py_function_error.6.query.sqlpp} | 12 +- .../query-ASTERIXDB-2857.1.ddl.sqlpp} | 30 +- .../query-ASTERIXDB-2857.2.update.sqlpp} | 6 +- .../query-ASTERIXDB-2857.3.query.sqlpp} | 23 +- .../in_let/in_let.8.query.sqlpp} | 21 +- .../synonym-03/synonym-03.1.ddl.sqlpp} | 19 +- .../synonym-03/synonym-03.2.query.sqlpp} | 9 +- .../api/cluster_state_1/cluster_state_1.1.regexadm | 2 + .../cluster_state_1_full.1.regexadm | 2 + .../cluster_state_1_less.1.regexadm | 2 + .../results/binary/query_id/query_id.1.adm | 6005 -------------------- .../custom-buffer-size/external_dataset.001.adm | 1 + .../custom-buffer-size/external_dataset.002.adm | 1 + .../custom-buffer-size/external_dataset.003.adm | 1 + .../custom-buffer-size/external_dataset.004.adm | 25 + .../custom-buffer-size/external_dataset.005.adm | 1 + .../custom-buffer-size/external_dataset.006.adm | 25 + .../py_function_error/py_function_error.3.json | 1 + .../py_function_error/py_function_error.4.json | 1 + .../query-ASTERIXDB-2857.3.adm | 2 + .../special_chars_2/special_chars_2.2.adm | 6 +- .../runtimets/results/subquery/in_let/in_let.8.adm | 2 + .../results/synonym/synonym-03/synonym-03.2.adm | 2 + .../resources/runtimets/testsuite_it_python.xml | 2 +- .../test/resources/runtimets/testsuite_sqlpp.xml | 41 +- .../asterix/common/api/IIdentifierMapper.java} | 11 +- .../asterix/common/config/CompilerProperties.java | 17 +- .../asterix/common/config/ExternalProperties.java | 8 +- .../common/config/OptimizationConfUtil.java | 15 + .../asterix/common/config/StorageProperties.java | 54 +- .../common/config/TransactionProperties.java | 5 +- .../asterix/common/exceptions/WarningUtil.java | 4 +- .../asterix/common/metadata/DataverseName.java | 74 +- .../common/utils/IdentifierMappingUtil.java} | 25 +- .../asterix/common/utils/IdentifierUtil.java} | 18 +- .../src/main/resources/asx_errormsg/en.properties | 4 +- .../asterix/common/metadata/DataverseNameTest.java | 25 +- .../asterix-doc/src/main/markdown/sqlpp/5_error.md | 2 +- .../src/main/markdown/sqlpp/7_ddl_dml.md | 3 +- .../apache/asterix/external/api/IRecordReader.java | 5 + .../dataflow/AbstractFeedDataFlowController.java | 5 + .../dataflow/FeedRecordDataFlowController.java | 6 + .../external/dataset/adapter/FeedAdapter.java | 5 + .../abstracts/AbstractExternalInputStream.java | 3 - .../AbstractExternalInputStreamFactory.java | 32 +- .../input/record/reader/aws/AwsS3InputStream.java | 82 +- .../record/reader/aws/AwsS3InputStreamFactory.java | 30 +- .../record/reader/azure/AzureBlobInputStream.java | 12 +- .../reader/azure/AzureBlobInputStreamFactory.java | 30 +- .../record/reader/stream/StreamRecordReader.java | 6 +- .../input/stream/AsterixInputStreamReader.java | 11 +- .../StandardUTF8ToModifiedUTF8DataOutput.java | 2 +- .../external/library/PythonLibraryEvaluator.java | 17 +- .../library/PythonLibraryEvaluatorFactory.java | 34 +- .../external/util/ExternalDataConstants.java | 14 +- .../asterix/external/util/ExternalDataUtils.java | 13 +- .../input/record/reader/awss3/AwsS3Test.java | 145 + .../org/apache/asterix/metadata/MetadataNode.java | 14 +- .../metadata/declared/DatasetDataSource.java | 6 + .../metadata/declared/MetadataManagerUtil.java | 8 +- .../metadata/declared/MetadataProvider.java | 17 +- .../apache/asterix/metadata/entities/Function.java | 7 + .../asterix/metadata/feeds/FeedMetadataUtil.java | 9 +- .../apache/asterix/metadata/utils/DatasetUtil.java | 6 +- .../InvertedIndexResourceFactoryProvider.java | 5 +- .../asterix/om/functions/BuiltinFunctions.java | 2 +- .../replication/management/NetworkingUtil.java | 16 +- .../replication/messaging/ReplicateFileTask.java | 7 + .../asterix/replication/sync/FileSynchronizer.java | 4 + .../functions/CreateQueryUIDDescriptor.java | 32 +- .../service/transaction/TransactionManager.java | 12 +- asterixdb/pom.xml | 2 +- .../appended-resources/supplemental-models.xml | 32 +- .../common/exceptions/AlgebricksException.java | 2 + .../api/HeuristicCompilerFactoryBuilder.java | 3 +- .../algebricks/algebricks-core/pom.xml | 2 +- .../logical/RunningAggregateOperator.java | 5 + .../algebra/util/OperatorManipulationUtil.java | 9 +- .../algebricks/core/config/AlgebricksConfig.java | 3 + .../algebricks/core/jobgen/impl/JobGenContext.java | 9 +- .../rewriter/base/PhysicalOptimizationConfig.java | 9 + .../rules/RemoveUnusedAssignAndAggregateRule.java | 15 +- .../hyracks/api/exceptions/HyracksException.java | 2 + .../api/exceptions/IFormattedException.java | 11 + .../org/apache/hyracks/api/exceptions/Warning.java | 27 +- .../org/apache/hyracks/api/util/InvokeUtil.java | 2 +- .../control/common/controllers/NCConfig.java | 4 + .../hyracks/hyracks-data/hyracks-data-std/pom.xml | 2 +- .../hyracks/dataflow/common/io/RunFileWriter.java | 10 +- .../hyracks/dataflow/std/join/NestedLoopJoin.java | 165 +- .../OptimizedHybridHashJoinOperatorDescriptor.java | 5 +- .../hyracks/http/server/AbstractServlet.java | 9 +- hyracks-fullstack/hyracks/hyracks-util/pom.xml | 2 +- hyracks-fullstack/pom.xml | 4 +- 155 files changed, 2427 insertions(+), 7219 deletions(-) diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java index b83fbdf,919a24e..ccd38d3 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java @@@ -134,9 -135,10 +135,10 @@@ public class APIFramework CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_WINDOWMEMORY_KEY, CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY, CompilerProperties.COMPILER_PARALLELISM_KEY, CompilerProperties.COMPILER_SORT_PARALLEL_KEY, CompilerProperties.COMPILER_SORT_SAMPLES_KEY, - CompilerProperties.COMPILER_INDEXONLY_KEY, CompilerProperties.COMPILER_INTERNAL_SANITYCHECK_KEY, + CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY, CompilerProperties.COMPILER_INDEXONLY_KEY, + CompilerProperties.COMPILER_INTERNAL_SANITYCHECK_KEY, CompilerProperties.COMPILER_EXTERNAL_FIELD_PUSHDOWN_KEY, CompilerProperties.COMPILER_SUBPLAN_MERGE_KEY, - CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, + CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, CompilerProperties.COMPILER_ARRAYINDEX_KEY, CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME, StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME, diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index ce6e10d,d8698f9..d2e9284 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@@ -18,7 -18,8 +18,9 @@@ */ package org.apache.asterix.app.translator; + import static org.apache.asterix.common.utils.IdentifierUtil.dataset; + import static org.apache.asterix.common.utils.IdentifierUtil.dataverse; +import static org.apache.asterix.lang.common.statement.CreateFullTextFilterStatement.FIELD_TYPE_STOPWORDS; import java.io.File; import java.io.FileInputStream; diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java index 8064c05,a58673f..5538a0a --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java @@@ -160,8 -165,8 +169,10 @@@ public class CompilerProperties extend public static final String COMPILER_MIN_MEMORY_ALLOCATION_KEY = Option.COMPILER_MIN_MEMORY_ALLOCATION.ini(); + public static final String COMPILER_ARRAYINDEX_KEY = Option.COMPILER_ARRAYINDEX.ini(); + + public static final String COMPILER_EXTERNALSCANMEMORY_KEY = Option.COMPILER_EXTERNALSCANMEMORY.ini(); + public static final int COMPILER_PARALLELISM_AS_STORAGE = 0; public CompilerProperties(PropertiesAccessor accessor) { @@@ -228,7 -233,7 +239,11 @@@ return accessor.getBoolean(Option.COMPILER_MIN_MEMORY_ALLOCATION); } + public boolean isArrayIndex() { + return accessor.getBoolean(Option.COMPILER_ARRAYINDEX); + } ++ + public int getExternalScanMemorySize() { + return accessor.getInt(Option.COMPILER_EXTERNALSCANMEMORY); + } } diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java index 81974fa,c4dbd85..d498d69 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java @@@ -72,9 -72,9 +72,12 @@@ public class OptimizationConfUtil CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, compilerProperties.getSubplanNestedPushdown()); boolean minMemoryAllocation = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, compilerProperties.getMinMemoryAllocation()); + boolean arrayIndex = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_ARRAYINDEX_KEY, + compilerProperties.isArrayIndex()); + int externalScanBufferSize = getExternalScanBufferSize( + (String) querySpecificConfig.get(CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY), + compilerProperties.getExternalScanMemorySize(), sourceLoc); + PhysicalOptimizationConfig physOptConf = new PhysicalOptimizationConfig(); physOptConf.setFrameSize(frameSize); physOptConf.setMaxFramesExternalSort(sortFrameLimit); @@@ -90,7 -90,7 +93,8 @@@ physOptConf.setSubplanMerge(subplanMerge); physOptConf.setSubplanNestedPushdown(subplanNestedPushdown); physOptConf.setMinMemoryAllocation(minMemoryAllocation); + physOptConf.setArrayIndexEnabled(arrayIndex); + physOptConf.setExternalScanBufferSize(externalScanBufferSize); return physOptConf; } diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java index 09709d0,3ec8aec..c769960 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java @@@ -19,7 -19,7 +19,8 @@@ package org.apache.asterix.metadata; +import static org.apache.asterix.common.exceptions.ErrorCode.FULL_TEXT_DEFAULT_CONFIG_CANNOT_BE_DELETED_OR_CREATED; + import static org.apache.asterix.common.utils.IdentifierUtil.dataset; import java.rmi.RemoteException; import java.util.ArrayList; diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java index 9241458,ec63825..497b15f --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java @@@ -618,10 -589,9 +620,10 @@@ public class MetadataProvider implement Index secondaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(), dataset.getDatasetName(), indexName); if (secondaryIndex == null) { - throw new AlgebricksException( - "Code generation error: no index " + indexName + " for dataset " + dataset.getDatasetName()); + throw new AlgebricksException("Code generation error: no index " + indexName + " for " + dataset() + " " + + dataset.getDatasetName()); } + Index.ValueIndexDetails secondaryIndexDetails = (Index.ValueIndexDetails) secondaryIndex.getIndexDetails(); RecordDescriptor outputRecDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context); Pair<IFileSplitProvider, AlgebricksPartitionConstraint> spPc = getSplitProviderAndConstraints(dataset, secondaryIndex.getIndexName()); diff --cc hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java index 944ddf7,817db5a..20632f1 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java @@@ -33,5 -34,6 +34,7 @@@ public class AlgebricksConfig public static final boolean SUBPLAN_MERGE_DEFAULT = true; public static final boolean SUBPLAN_NESTEDPUSHDOWN_DEFAULT = true; public static final boolean MIN_MEMORY_ALLOCATION_DEFAULT = true; + public static final boolean ARRAY_INDEX_DEFAULT = false; + public static final int EXTERNAL_SCAN_BUFFER_SIZE = + StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.KILOBYTE); } diff --cc hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java index f18cad2,bfff925..62bf671 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java @@@ -45,7 -45,7 +45,8 @@@ public class PhysicalOptimizationConfi private static final String SUBPLAN_MERGE = "SUBPLAN_MERGE"; private static final String SUBPLAN_NESTEDPUSHDOWN = "SUBPLAN_NESTEDPUSHDOWN"; private static final String MIN_MEMORY_ALLOCATION = "MIN_MEMORY_ALLOCATION"; + private static final String ARRAY_INDEX = "ARRAY_INDEX"; + private static final String EXTERNAL_SCAN_BUFFER_SIZE = "EXTERNAL_SCAN_BUFFER_SIZE"; private final Properties properties = new Properties(); @@@ -227,14 -227,14 +228,22 @@@ setBoolean(MIN_MEMORY_ALLOCATION, value); } + public boolean isArrayIndexEnabled() { + return getBoolean(ARRAY_INDEX, AlgebricksConfig.ARRAY_INDEX_DEFAULT); + } + + public void setArrayIndexEnabled(boolean arrayIndex) { + setBoolean(ARRAY_INDEX, arrayIndex); + } + + public int getExternalScanBufferSize() { + return getInt(EXTERNAL_SCAN_BUFFER_SIZE, AlgebricksConfig.EXTERNAL_SCAN_BUFFER_SIZE); + } + + public void setExternalScanBufferSize(int bufferSize) { + setInt(EXTERNAL_SCAN_BUFFER_SIZE, bufferSize); + } + private void setInt(String property, int value) { properties.setProperty(property, Integer.toString(value)); }
