Yingyi Bu has submitted this change and it was merged. Change subject: ASTERIXDB-1760: support per-query customizations for compilation parameters. ......................................................................
ASTERIXDB-1760: support per-query customizations for compilation parameters. Change-Id: I6e18f3f7706e574553a02e15a39daddda3c413b2 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1458 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> BAD: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java M asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_load_20M/big_object_load_20M.2.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.3.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.3.query.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java 10 files changed, 245 insertions(+), 11 deletions(-) Approvals: Till Westmann: Looks good to me, approved Jenkins: Verified; No violations found; No violations found; Verified diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java index 4140f9a..90a8599 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java @@ -35,7 +35,9 @@ import org.apache.asterix.app.result.ResultUtil; import org.apache.asterix.common.config.CompilerProperties; import org.apache.asterix.common.config.ExternalProperties; +import org.apache.asterix.common.config.IPropertyInterpreter; import org.apache.asterix.common.config.OptimizationConfUtil; +import org.apache.asterix.common.config.PropertyInterpreters; import org.apache.asterix.common.exceptions.ACIDException; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.compiler.provider.ILangCompilationProvider; @@ -59,9 +61,9 @@ import org.apache.asterix.runtime.job.listener.JobEventListenerFactory; import org.apache.asterix.runtime.util.AppContextInfo; import org.apache.asterix.transaction.management.service.transaction.JobIdFactory; +import org.apache.asterix.translator.SessionConfig; import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement; import org.apache.asterix.translator.IStatementExecutor.Stats; -import org.apache.asterix.translator.SessionConfig; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; @@ -208,9 +210,13 @@ CompilerProperties compilerProperties = AppContextInfo.INSTANCE.getCompilerProperties(); int frameSize = compilerProperties.getFrameSize(); - int sortFrameLimit = (int) (compilerProperties.getSortMemorySize() / frameSize); - int groupFrameLimit = (int) (compilerProperties.getGroupMemorySize() / frameSize); - int joinFrameLimit = (int) (compilerProperties.getJoinMemorySize() / frameSize); + Map<String, String> querySpecificConfig = metadataProvider.getConfig(); + int sortFrameLimit = getFrameLimit(querySpecificConfig.get(CompilerProperties.COMPILER_SORTMEMORY_KEY), + compilerProperties.getSortMemorySize(), frameSize); + int groupFrameLimit = getFrameLimit(querySpecificConfig.get(CompilerProperties.COMPILER_GROUPMEMORY_KEY), + compilerProperties.getGroupMemorySize(), frameSize); + int joinFrameLimit = getFrameLimit(querySpecificConfig.get(CompilerProperties.COMPILER_JOINMEMORY_KEY), + compilerProperties.getJoinMemorySize(), frameSize); OptimizationConfUtil.getPhysicalOptimizationConfig().setFrameSize(frameSize); OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesExternalSort(sortFrameLimit); OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesExternalGroupBy(groupFrameLimit); @@ -230,7 +236,8 @@ builder.setMissableTypeComputer(MissableTypeComputer.INSTANCE); builder.setConflictingTypeResolver(ConflictingTypeResolver.INSTANCE); - int parallelism = compilerProperties.getParallelism(); + int parallelism = getParallelism(querySpecificConfig.get(CompilerProperties.COMPILER_PARALLELISM_KEY), + compilerProperties.getParallelism()); builder.setClusterLocations(parallelism == CompilerProperties.COMPILER_PARALLELISM_AS_STORAGE ? metadataProvider.getClusterLocations() : getComputationLocations(clusterInfoCollector, parallelism)); @@ -404,4 +411,19 @@ throw new AlgebricksException(e); } } + + // Gets the frame limit. + private int getFrameLimit(String parameter, long memBudgetInConfiguration, int frameSize) { + IPropertyInterpreter<Long> longBytePropertyInterpreter = PropertyInterpreters.getLongBytePropertyInterpreter(); + long memBudget = parameter == null ? memBudgetInConfiguration + : longBytePropertyInterpreter.interpret(parameter); + return (int) (memBudget / frameSize); + } + + // Gets the parallelism parameter. + private int getParallelism(String parameter, int parallelismInConfiguration) { + IPropertyInterpreter<Integer> integerIPropertyInterpreter = PropertyInterpreters + .getIntegerPropertyInterpreter(); + return parameter == null ? parallelismInConfiguration : integerIPropertyInterpreter.interpret(parameter); + } } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_load_20M/big_object_load_20M.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_load_20M/big_object_load_20M.2.update.aql index f384e96..39842e4 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_load_20M/big_object_load_20M.2.update.aql +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_load_20M/big_object_load_20M.2.update.aql @@ -26,8 +26,8 @@ use dataverse testdv2; -set hash_merge "true" +set "compiler.sortmemory" "64MB" load dataset testds using localfs -(("path"="asterix_nc1://target/data/big-object/big_object_20M.adm"),("format"="adm")) pre-sorted; +(("path"="asterix_nc1://target/data/big-object/big_object_20M.adm"),("format"="adm")); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.1.ddl.sqlpp new file mode 100644 index 0000000..e81fa19 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.1.ddl.sqlpp @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Test case Name : big_object_bulkload.aql + * Description : bulkload insert of large objects + * Expected Result : Success + * Date : 20th April 2016 + */ + +drop dataverse testdv2 if exists; +create dataverse testdv2; +use testdv2; + +create type testtype as closed { + id: int64, + name: string, + hobbies: {{string}} +} + +create dataset testds(testtype) primary key id; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.2.update.sqlpp new file mode 100644 index 0000000..09f3a69 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.2.update.sqlpp @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * + * Big object (20 MB) loading test + * Expected result: success + * + */ + +use testdv2; + +set `compiler.sortmemory` "64MB" + +load dataset testds +using localfs +(("path"="asterix_nc1://target/data/big-object/big_object_20M.adm"),("format"="adm")); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.3.query.sqlpp new file mode 100644 index 0000000..4730ac8 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.3.query.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use testdv2; + +select value d +from testds d +where d.id=1; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.1.ddl.sqlpp new file mode 100644 index 0000000..92698ab --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.1.ddl.sqlpp @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse tpch if exists; +create dataverse tpch; + +use tpch; + + +create type tpch.LineItemType as + closed { + l_orderkey : bigint, + l_partkey : bigint, + l_suppkey : bigint, + l_linenumber : bigint, + l_quantity : double, + l_extendedprice : double, + l_discount : double, + l_tax : double, + l_returnflag : string, + l_linestatus : string, + l_shipdate : string, + l_commitdate : string, + l_receiptdate : string, + l_shipinstruct : string, + l_shipmode : string, + l_comment : string +} + +create dataset LineItem(LineItemType) primary key l_orderkey,l_linenumber; + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.2.update.sqlpp new file mode 100644 index 0000000..5fe734c --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.2.update.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use tpch; + + +load dataset LineItem using localfs ((`path`=`asterix_nc1://data/tpch0.001/lineitem.tbl`),(`format`=`delimited-text`),(`delimiter`=`|`)) pre-sorted; + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.3.query.sqlpp new file mode 100644 index 0000000..a4876a2 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.3.query.sqlpp @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE tpch; + +SET `compiler.parallelism` "5" + +SELECT l_returnflag, + l_linestatus, + sum(l_quantity) AS sum_qty, + sum(l_extendedprice) AS sum_base_price, + sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price, + sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge, + avg(l_quantity) AS ave_qty, + avg(l_extendedprice) AS ave_price, + avg(l_discount) AS ave_disc, + count(1) AS count_order +FROM LineItem +WHERE l_shipdate <= '1998-09-02' +/* +hash */ +GROUP BY l_returnflag, l_linestatus +ORDER BY l_returnflag, l_linestatus +; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index 0aaf8c3..9bd2126 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -6822,6 +6822,11 @@ </compilation-unit> </test-case> <test-case FilePath="tpch-sql-sugar"> + <compilation-unit name="q01_pricing_summary_report_parallelism"> + <output-dir compare="Text">q01_pricing_summary_report_nt</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="tpch-sql-sugar"> <compilation-unit name="q20_potential_part_promotion"> <output-dir compare="Text">q20_potential_part_promotion</output-dir> </compilation-unit> @@ -7824,6 +7829,11 @@ <output-dir compare="Text">big_object_join</output-dir> </compilation-unit> </test-case> + <test-case FilePath="big-object"> + <compilation-unit name="big_object_load_20M"> + <output-dir compare="Text">big_object_load_20M</output-dir> + </compilation-unit> + </test-case> </test-group> <test-group name="external-indexing"> <test-case FilePath="external-indexing"> diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java index 4e18bde..3710e54 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java @@ -25,19 +25,19 @@ public class CompilerProperties extends AbstractProperties { - private static final String COMPILER_SORTMEMORY_KEY = "compiler.sortmemory"; + public static final String COMPILER_SORTMEMORY_KEY = "compiler.sortmemory"; private static final long COMPILER_SORTMEMORY_DEFAULT = StorageUtil.getSizeInBytes(32, MEGABYTE); - private static final String COMPILER_GROUPMEMORY_KEY = "compiler.groupmemory"; + public static final String COMPILER_GROUPMEMORY_KEY = "compiler.groupmemory"; private static final long COMPILER_GROUPMEMORY_DEFAULT = StorageUtil.getSizeInBytes(32, MEGABYTE); - private static final String COMPILER_JOINMEMORY_KEY = "compiler.joinmemory"; + public static final String COMPILER_JOINMEMORY_KEY = "compiler.joinmemory"; private static final long COMPILER_JOINMEMORY_DEFAULT = StorageUtil.getSizeInBytes(32, MEGABYTE); private static final String COMPILER_FRAMESIZE_KEY = "compiler.framesize"; private static final int COMPILER_FRAMESIZE_DEFAULT = StorageUtil.getSizeInBytes(32, KILOBYTE); - private static final String COMPILER_PARALLELISM_KEY = "compiler.parallelism"; + public static final String COMPILER_PARALLELISM_KEY = "compiler.parallelism"; public static final int COMPILER_PARALLELISM_AS_STORAGE = 0; private static final String COMPILER_PREGELIX_HOME = "compiler.pregelix.home"; -- To view, visit https://asterix-gerrit.ics.uci.edu/1458 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I6e18f3f7706e574553a02e15a39daddda3c413b2 Gerrit-PatchSet: 4 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Yingyi Bu <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: Yingyi Bu <[email protected]>
