This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 70ff690f9acb8d4eb8ba478d1f95e98b952a2ef7 Author: shivajah <[email protected]> AuthorDate: Sun Feb 23 20:36:20 2020 -0800 [NO ISSUE] Fix error message for low memory in hash join - user model changes: no - storage format changes: no - interface changes: no Change-Id: Icebdfd82beede2cd86e178e118ee2dd8e97edc77 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/4946 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> --- .../big_object_join.1.ddl.sqlpp | 82 ++++++++++++++++++++++ .../big_object_join.2.update.sqlpp | 24 +++++++ .../big_object_join.3.query.sqlpp | 34 +++++++++ .../test/resources/runtimets/testsuite_sqlpp.xml | 7 ++ .../apache/hyracks/api/exceptions/ErrorCode.java | 1 + .../src/main/resources/errormsg/en.properties | 1 + .../dataflow/std/join/OptimizedHybridHashJoin.java | 4 +- 7 files changed, 151 insertions(+), 2 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_join_low_memory_err/big_object_join.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_join_low_memory_err/big_object_join.1.ddl.sqlpp new file mode 100644 index 0000000..80d8d19 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_join_low_memory_err/big_object_join.1.ddl.sqlpp @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* +* Description : Create an external dataset that contains long comments fields, 10% of the records have a 32K size comments. +* This will trigger into the VSizeFrame path +* Expected Res : Success +* Date : Jun 16 2015 +*/ + +drop dataverse test if exists; +create dataverse test; + +use test; + + +create type test.LineType as + closed { + l_orderkey : integer, + l_partkey : integer, + l_suppkey : integer, + l_linenumber : integer, + l_quantity : double, + l_extendedprice : double, + l_discount : double, + l_tax : double, + l_returnflag : string, + l_linestatus : string, + l_shipdate : string, + l_commitdate : string, + l_receiptdate : string, + l_shipinstruct : string, + l_shipmode : string, + l_comment : string +}; + +create type test.OrderType as + closed { + o_orderkey : integer, + o_custkey : integer, + o_orderstatus : string, + o_totalprice : double, + o_orderdate : string, + o_orderpriority : string, + o_clerk : string, + o_shippriority : integer, + o_comment : string +}; + +create type test.CustomerType as + closed { + c_custkey : integer, + c_name : string, + c_address : string, + c_nationkey : integer, + c_phone : string, + c_acctbal : double, + c_mktsegment : string, + c_comment : string +}; + +create external dataset Line(LineType) using `localfs`((`path`=`asterix_nc1://data/big-object/lineitem.tbl.big`),(`input-format`=`text-input-format`),(`format`=`delimited-text`),(`delimiter`=`|`)); + +create external dataset `Order`(OrderType) using `localfs`((`path`=`asterix_nc1://data/big-object/order.tbl.verylong.big`),(`input-format`=`text-input-format`),(`format`=`delimited-text`),(`delimiter`=`|`)); + +create external dataset Customer(CustomerType) using `localfs`((`path`=`asterix_nc1://data/big-object/customer.tbl.big`),(`input-format`=`text-input-format`),(`format`=`delimited-text`),(`delimiter`=`|`)); + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_join_low_memory_err/big_object_join.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_join_low_memory_err/big_object_join.2.update.sqlpp new file mode 100644 index 0000000..00a13e7 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_join_low_memory_err/big_object_join.2.update.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* +* Description : Create an external dataset that contains long comments fields, 10% of the records have a 32K size comments. +* This will trigger into the VSizeFrame path +* Expected Res : Success +* Date : Jun 16 2015 +*/ diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_join_low_memory_err/big_object_join.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_join_low_memory_err/big_object_join.3.query.sqlpp new file mode 100644 index 0000000..0dd36b8 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_join_low_memory_err/big_object_join.3.query.sqlpp @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* +* Description : Create an external dataset that contains long comments fields, 10% of the records have a 32K size comments. + This will trigger into the VSizeFrame path +* Expected Res : Success +* Date : Jun 16 2015 +*/ +SET `compiler.joinmemory` "160KB"; +use test; + + +select element {'c_custkey':c.c_custkey,'o_orderkey':o.o_orderkey,'len_c_comment':test.`string-length`(c.c_comment),'len_o_comment':test.`string-length`(o.o_comment),'c_comment':c.c_comment} +from Customer as c, + `Order` as o +where (c.c_custkey = o.o_custkey) +order by o.o_orderkey, c.c_custkey +; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index 461e164..16c411d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -11538,6 +11538,13 @@ </compilation-unit> </test-case> <test-case FilePath="big-object"> + <compilation-unit name="big_object_join_low_memory_err"> + <output-dir compare="Text">big_object_join</output-dir> + <expected-error>HYR0123: Insufficient memory is provided for the join operators, please increase the join memory budget.</expected-error> + <source-location>false</source-location> + </compilation-unit> + </test-case> + <test-case FilePath="big-object"> <compilation-unit name="big_object_load_20M"> <output-dir compare="Text">big_object_load_20M</output-dir> </compilation-unit> diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java index 5716262..38f49dc 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java @@ -156,6 +156,7 @@ public class ErrorCode { public static final int EOF = 120; public static final int NUMERIC_PROMOTION_ERROR = 121; public static final int ERROR_PRINTING_PLAN = 122; + public static final int INSUFFICIENT_MEMORY = 123; // Compilation error codes. public static final int RULECOLLECTION_NOT_INSTANCE_OF_LIST = 10000; diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties index fcbb6bb..cfb6c9e 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties @@ -139,6 +139,7 @@ 120 = End of file 121 = A numeric type promotion error has occurred: %1$s 122 = Encountered an error while printing the plan +123 = Insufficient memory is provided for the join operators, please increase the join memory budget. 10000 = The given rule collection %1$s is not an instance of the List class. 10001 = Cannot compose partition constraint %1$s with %2$s diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java index 9c28c61..68b4b7b 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java @@ -31,6 +31,7 @@ import org.apache.hyracks.api.dataflow.value.IPredicateEvaluator; import org.apache.hyracks.api.dataflow.value.ITuplePairComparator; import org.apache.hyracks.api.dataflow.value.ITuplePartitionComputer; import org.apache.hyracks.api.dataflow.value.RecordDescriptor; +import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor; @@ -150,8 +151,7 @@ public class OptimizedHybridHashJoin { while (!bufferManager.insertTuple(pid, accessorBuild, tid, tempPtr)) { int victimPartition = spillPolicy.selectVictimPartition(pid); if (victimPartition < 0) { - throw new HyracksDataException( - "No more space left in the memory buffer, please assign more memory to hash-join."); + throw HyracksDataException.create(ErrorCode.INSUFFICIENT_MEMORY); } spillPartition(victimPartition); }
