This is an automated email from the ASF dual-hosted git repository. imaxon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 2cea7d8c915c43d598f6fc56536617b1eca1ed1d Author: Ali Alsuliman <[email protected]> AuthorDate: Thu Jan 15 18:19:36 2026 -0800 [ASTERIXDB-3692][COMP] Allow setting frame size at query level - user model changes: no - storage format changes: no - interface changes: no Details: 'compiler.framesize' is currently only a service-level configurable parameter, so changing the frame size requires a restart to reflect the frame size across all components. Very often, users want to try different frame sizes for a query. This patch is to allow users to set the frame size at query level. Note that there are still some parts of the query processing that is still using the service-level frame size such as the frame allocated for reading the query result. Ext-ref: MB-66320 Change-Id: Icf52985cd22eed8ae93439d1faa187ce23d31630 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20801 Reviewed-by: Murtadha Hubail <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> --- .../provider/SqlppCompilationProvider.java | 2 +- .../src/test/resources/runtimets/sqlpp_queries.xml | 8 ++--- .../asterix/common/config/CompilerProperties.java | 5 ++- .../common/config/OptimizationConfUtil.java | 42 ++++++++++++++-------- .../asterix/common/exceptions/ErrorCode.java | 1 + .../src/main/resources/asx_errormsg/en.properties | 1 + .../metadata/utils/SampleOperationsHelper.java | 2 +- .../utils/SecondaryIndexOperationsHelper.java | 3 +- 8 files changed, 42 insertions(+), 22 deletions(-) diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java index ca97aa6734..b6a9295c14 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java @@ -95,7 +95,7 @@ public class SqlppCompilationProvider implements ILangCompilationProvider { CompilerProperties.COMPILER_CBO_TEST_KEY, CompilerProperties.COMPILER_FORCE_JOIN_ORDER_KEY, CompilerProperties.COMPILER_QUERY_PLAN_SHAPE_KEY, CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, CompilerProperties.COMPILER_COLUMN_FILTER_KEY, CompilerProperties.COMPILER_BATCH_LOOKUP_KEY, - FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, + CompilerProperties.COMPILER_FRAMESIZE_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, CompilerProperties.COMPILER_MAX_VARIABLE_OCCURRENCES_INLINING_KEY, CompilerProperties.COMPILER_DELTALAKE_FILESPLITS_KEY, FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME, StartFeedStatement.WAIT_FOR_COMPLETION, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml index 57b494abc9..bc1834fefe 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml @@ -12866,7 +12866,7 @@ <test-case FilePath="tpch-sql-sugar"> <compilation-unit name="q17_large_gby_variant_parameter"> <output-dir compare="Text">none</output-dir> - <expected-error>Invalid query parameter compiler.groupmemory -- value has to be greater than or equal to</expected-error> + <expected-error>Invalid `compiler.groupmemory` "-10737418240" for frame size=32768. value should be >= 4 * frame size: `compiler.groupmemory` "131072" in bytes</expected-error> </compilation-unit> </test-case> <test-case FilePath="tpch-sql-sugar"> @@ -12897,7 +12897,7 @@ <test-case FilePath="tpch-sql-sugar"> <compilation-unit name="q01_pricing_summary_report_parameter"> <output-dir compare="Text">none</output-dir> - <expected-error>Invalid query parameter compiler.sortmemory -- value has to be greater than or equal to</expected-error> + <expected-error>Invalid `compiler.sortmemory` "-7516192768" for frame size=32768. value should be >= 3 * frame size: `compiler.sortmemory` "98304" in bytes</expected-error> </compilation-unit> </test-case> <test-case FilePath="tpch-sql-sugar"> @@ -12958,7 +12958,7 @@ <test-case FilePath="tpch-sql-sugar"> <compilation-unit name="q09_product_type_profit_parameter"> <output-dir compare="Text">none</output-dir> - <expected-error>Invalid query parameter compiler.joinmemory -- value has to be greater than or equal to</expected-error> + <expected-error>Invalid `compiler.joinmemory` "-34359738368" for frame size=32768. value should be >= 5 * frame size: `compiler.joinmemory` "163840" in bytes</expected-error> </compilation-unit> </test-case> <test-case FilePath="tpch-sql-sugar"> @@ -16563,7 +16563,7 @@ <output-dir compare="Text">misc_01</output-dir> <expected-error>ASX0002: Type mismatch</expected-error> <expected-error>ASX1104: Invalid modifier FROM FIRST/LAST for function</expected-error> - <expected-error>ASX1037: Invalid query parameter compiler.windowmemory</expected-error> + <expected-error>Invalid `compiler.windowmemory` "102400" for frame size=32768. value should be >= 5 * frame size: `compiler.windowmemory` "163840" in bytes </expected-error> <expected-error>ASX1102: Expected window or aggregate function, got: lowercase</expected-error> <expected-error>ASX1079: Compilation error: count is a SQL-92 aggregate function</expected-error> <expected-error>ASX1104: Invalid modifier RESPECT/IGNORE NULLS for function</expected-error> diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java index da5f39b954..62be4d02c3 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java @@ -28,6 +28,7 @@ import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER_BYTE_ import static org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT; import static org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER; import static org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER; +import static org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER_BYTE_UNIT; import static org.apache.hyracks.control.common.config.OptionTypes.STRING; import static org.apache.hyracks.control.common.config.OptionTypes.getRangedIntegerType; import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE; @@ -67,7 +68,7 @@ public class CompilerProperties extends AbstractProperties { StorageUtil.getIntSizeInBytes(8, KILOBYTE), "The memory budget (in bytes) for an external scan operator instance in a partition"), COMPILER_FRAMESIZE( - INTEGER_BYTE_UNIT, + POSITIVE_INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(32, KILOBYTE), "The page size (in bytes) for computation"), COMPILER_MIN_SORTMEMORY( @@ -196,6 +197,8 @@ public class CompilerProperties extends AbstractProperties { } } + public static final String COMPILER_FRAMESIZE_KEY = Option.COMPILER_FRAMESIZE.ini(); + public static final String COMPILER_SORTMEMORY_KEY = Option.COMPILER_SORTMEMORY.ini(); public static final String COMPILER_GROUPMEMORY_KEY = Option.COMPILER_GROUPMEMORY.ini(); diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java index 6de87a9013..9ebf5c2dc3 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java @@ -51,8 +51,8 @@ public class OptimizationConfUtil { public static PhysicalOptimizationConfig createPhysicalOptimizationConf(CompilerProperties compilerProperties, Map<String, Object> querySpecificConfig, Set<String> parameterNames, SourceLocation sourceLoc) throws AlgebricksException { - int frameSize = compilerProperties.getFrameSize(); - int sortFrameLimit = getSortNumFrames(compilerProperties, querySpecificConfig, sourceLoc); + int frameSize = getFrameSize(compilerProperties, querySpecificConfig, sourceLoc); + int sortFrameLimit = getSortNumFrames(compilerProperties, querySpecificConfig, sourceLoc, frameSize); int groupFrameLimit = getFrameLimit(CompilerProperties.COMPILER_GROUPMEMORY_KEY, (String) querySpecificConfig.get(CompilerProperties.COMPILER_GROUPMEMORY_KEY), compilerProperties.getGroupMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_GROUP_BY, sourceLoc); @@ -62,7 +62,8 @@ public class OptimizationConfUtil { int windowFrameLimit = getFrameLimit(CompilerProperties.COMPILER_WINDOWMEMORY_KEY, (String) querySpecificConfig.get(CompilerProperties.COMPILER_WINDOWMEMORY_KEY), compilerProperties.getWindowMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_WINDOW, sourceLoc); - int textSearchFrameLimit = getTextSearchNumFrames(compilerProperties, querySpecificConfig, sourceLoc); + int textSearchFrameLimit = + getTextSearchNumFrames(compilerProperties, frameSize, querySpecificConfig, sourceLoc); int sortNumSamples = getSortSamples(compilerProperties, querySpecificConfig, sourceLoc); boolean fullParallelSort = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_SORT_PARALLEL_KEY, compilerProperties.getSortParallel()); @@ -143,6 +144,21 @@ public class OptimizationConfUtil { return physOptConf; } + private static int getFrameSize(CompilerProperties compilerProperties, Map<String, Object> querySpecificConfig, + SourceLocation sourceLoc) throws AlgebricksException { + String queryFrameSize = (String) querySpecificConfig.get(CompilerProperties.COMPILER_FRAMESIZE_KEY); + if (queryFrameSize == null) { + return compilerProperties.getFrameSize(); + } + @SuppressWarnings("unchecked") + IOptionType<Integer> type = CompilerProperties.Option.COMPILER_FRAMESIZE.type(); + try { + return type.parse(queryFrameSize); + } catch (IllegalArgumentException e) { + throw AsterixException.create(ErrorCode.COMPILATION_ERROR, e, sourceLoc, e.getMessage()); + } + } + private static int getExternalScanBufferSize(String externalScanMemorySizeParameter, int compilerExternalScanMemorySize, SourceLocation sourceLoc) throws AsterixException { IOptionType<Integer> intByteParser = OptionTypes.INTEGER_BYTE_UNIT; @@ -155,27 +171,25 @@ public class OptimizationConfUtil { } public static int getSortNumFrames(CompilerProperties compilerProperties, Map<String, Object> querySpecificConfig, - SourceLocation sourceLoc) throws AlgebricksException { + SourceLocation sourceLoc, int frameSize) throws AlgebricksException { return getFrameLimit(CompilerProperties.COMPILER_SORTMEMORY_KEY, (String) querySpecificConfig.get(CompilerProperties.COMPILER_SORTMEMORY_KEY), - compilerProperties.getSortMemorySize(), compilerProperties.getFrameSize(), MIN_FRAME_LIMIT_FOR_SORT, - sourceLoc); + compilerProperties.getSortMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_SORT, sourceLoc); } public static int getGroupByNumFrames(CompilerProperties compilerProperties, - Map<String, Object> querySpecificConfig, SourceLocation sourceLoc) throws AlgebricksException { + Map<String, Object> querySpecificConfig, SourceLocation sourceLoc, int frameSize) + throws AlgebricksException { return getFrameLimit(CompilerProperties.COMPILER_GROUPMEMORY_KEY, (String) querySpecificConfig.get(CompilerProperties.COMPILER_GROUPMEMORY_KEY), - compilerProperties.getGroupMemorySize(), compilerProperties.getFrameSize(), - MIN_FRAME_LIMIT_FOR_GROUP_BY, sourceLoc); + compilerProperties.getGroupMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_GROUP_BY, sourceLoc); } - public static int getTextSearchNumFrames(CompilerProperties compilerProperties, + public static int getTextSearchNumFrames(CompilerProperties compilerProperties, int frameSize, Map<String, Object> querySpecificConfig, SourceLocation sourceLoc) throws AlgebricksException { return getFrameLimit(CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY, (String) querySpecificConfig.get(CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY), - compilerProperties.getTextSearchMemorySize(), compilerProperties.getFrameSize(), - MIN_FRAME_LIMIT_FOR_TEXT_SEARCH, sourceLoc); + compilerProperties.getTextSearchMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_TEXT_SEARCH, sourceLoc); } @SuppressWarnings("squid:S1166") // Either log or rethrow this exception @@ -190,8 +204,8 @@ public class OptimizationConfUtil { } int frameLimit = (int) (memBudget / frameSize); if (frameLimit < minFrameLimit) { - throw AsterixException.create(ErrorCode.COMPILATION_BAD_QUERY_PARAMETER_VALUE, sourceLoc, parameterName, - frameSize * minFrameLimit, "bytes"); + throw AsterixException.create(ErrorCode.INVALID_FRAME_BASED_MEMORY_BUDGET, sourceLoc, parameterName, + memBudget, frameSize, minFrameLimit, frameSize * minFrameLimit, "bytes"); } // sets the frame limit to the minimum frame limit if the calculated frame limit is too small. return Math.max(frameLimit, minFrameLimit); diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index f604f6628e..d52fe0799b 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@ -346,6 +346,7 @@ public enum ErrorCode implements IError { ICEBERG_SNAPSHOT_ID_NOT_FOUND(1238), INVALID_ICEBERG_SNAPSHOT_VALUE(1239), NO_VALID_CREDENTIALS_PROVIDED_FOR_BIGLAKE_METASTORE_CATALOG(1240), + INVALID_FRAME_BASED_MEMORY_BUDGET(1241), // Feed errors DATAFLOW_ILLEGAL_STATE(3001), diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index 16c5c99fe5..4b0fd2bd41 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -348,6 +348,7 @@ 1238 = Iceberg snapshot '%1$s' not found 1239 = Invalid Iceberg snapshot value: '%1$s' 1240 = No valid credentials provided to access Biglake Metastore catalog. +1241 = Invalid `%1$s` "%2$s" for frame size=%3$s. value should be >= %4$s * frame size: `%1$s` "%5$s" in %6$s # Feed Errors 3001 = Illegal state. diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SampleOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SampleOperationsHelper.java index 7e3af78b59..b14e0e574b 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SampleOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SampleOperationsHelper.java @@ -380,6 +380,6 @@ public class SampleOperationsHelper implements ISecondaryIndexOperationsHelper { throws AlgebricksException { return OptimizationConfUtil.getGroupByNumFrames( metadataProvider.getApplicationContext().getCompilerProperties(), metadataProvider.getConfig(), - sourceLoc); + sourceLoc, metadataProvider.getApplicationContext().getCompilerProperties().getFrameSize()); } } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java index 8350141153..b92dc2815d 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java @@ -158,7 +158,8 @@ public abstract class SecondaryIndexOperationsHelper implements ISecondaryIndexO private static int getSortNumFrames(MetadataProvider metadataProvider, SourceLocation sourceLoc) throws AlgebricksException { return OptimizationConfUtil.getSortNumFrames(metadataProvider.getApplicationContext().getCompilerProperties(), - metadataProvider.getConfig(), sourceLoc); + metadataProvider.getConfig(), sourceLoc, + metadataProvider.getApplicationContext().getCompilerProperties().getFrameSize()); } public static ISecondaryIndexOperationsHelper createIndexOperationsHelper(Dataset dataset, Index index,
