Repository: incubator-impala Updated Branches: refs/heads/master 7c368999f -> d04f96b99
IMPALA-5301: Set Kudu minicluster memory limit By default, Kudu assumes it has 80% of system memory which is far too high for the minicluster. This sets a mem limit of 2gb and lowers the limit of the block cache. These values were tested on a gerrit-verify-dryrun job as well as an exhaustive run. This patch also simplifies TestKuduMemLimits which was unnecessarily creating a large table during test execution. Change-Id: I7fd7e1cd9dc781aaa672a2c68c845cb57ec885d5 Reviewed-on: http://gerrit.cloudera.org:8080/6844 Reviewed-by: Todd Lipcon <[email protected]> Reviewed-by: Tim Armstrong <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/d04f96b9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/d04f96b9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/d04f96b9 Branch: refs/heads/master Commit: d04f96b990c18ed17c82c0151978b56f936f0574 Parents: 7c36899 Author: Matthew Jacobs <[email protected]> Authored: Wed May 10 11:10:21 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Wed May 17 23:44:37 2017 +0000 ---------------------------------------------------------------------- .../node_templates/cdh5/etc/init.d/kudu-common | 2 + tests/query_test/test_kudu.py | 45 ++------------------ 2 files changed, 5 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d04f96b9/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-common ---------------------------------------------------------------------- diff --git a/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-common b/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-common index 2d8b5c3..a8d2d4f 100644 --- a/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-common +++ b/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-common @@ -41,6 +41,8 @@ KUDU_COMMON_ARGS=("-webserver_doc_root=$KUDU_WWW_DIR") # Add a dummy flag with value IBelongToTheMiniCluster so the process is easily # identifiable in case the pid file is removed. KUDU_COMMON_ARGS+=("-vmodule=IBelongToTheMiniCluster") +KUDU_COMMON_ARGS+=("-memory_limit_hard_bytes=2147483648") # 2gb +KUDU_COMMON_ARGS+=("-block_cache_capacity_mb=256") HOLE_PUNCH_TEST_FILE=${NODE_DIR}/var/lib/kudu/hole_punch_test if ! fallocate -p -o 1 -l 1 "$HOLE_PUNCH_TEST_FILE" &>/dev/null; then http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d04f96b9/tests/query_test/test_kudu.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py index 8c80ad8..7bfdf4b 100644 --- a/tests/query_test/test_kudu.py +++ b/tests/query_test/test_kudu.py @@ -789,54 +789,15 @@ class TestImpalaKuduIntegration(KuduTestSuite): class TestKuduMemLimits(KuduTestSuite): - QUERIES = ["select * from lineitem where l_orderkey = -1", - "select * from lineitem where l_commitdate like '%cheese'", - "select * from lineitem limit 90"] + QUERIES = ["select * from tpch_kudu.lineitem where l_orderkey = -1", + "select * from tpch_kudu.lineitem where l_commitdate like '%cheese'", + "select * from tpch_kudu.lineitem limit 90"] # The value indicates the minimum memory requirements for the queries above, the first # memory limit corresponds to the first query QUERY_MEM_LIMITS = [1, 1, 10] - CREATE = """ - CREATE TABLE lineitem ( - l_orderkey BIGINT, - l_linenumber INT, - l_partkey BIGINT, - l_suppkey BIGINT, - l_quantity double, - l_extendedprice double, - l_discount double, - l_tax double, - l_returnflag STRING, - l_linestatus STRING, - l_shipdate STRING, - l_commitdate STRING, - l_receiptdate STRING, - l_shipinstruct STRING, - l_shipmode STRING, - l_comment STRING, - PRIMARY KEY (l_orderkey, l_linenumber)) - PARTITION BY HASH (l_orderkey, l_linenumber) PARTITIONS 3 - STORED AS KUDU""" - - LOAD = """ - insert into lineitem - select l_orderkey, l_linenumber, l_partkey, l_suppkey, cast(l_quantity as double), - cast(l_extendedprice as double), cast(l_discount as double), cast(l_tax as double), - l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, - l_shipmode, l_comment from tpch_parquet.lineitem""" - - @classmethod - def auto_create_db(cls): - return True - - @pytest.fixture(scope='class') - def test_data(cls, cls_cursor): - cls_cursor.execute(cls.CREATE) - cls_cursor.execute(cls.LOAD) - @pytest.mark.execute_serially - @pytest.mark.usefixtures("test_data") @pytest.mark.parametrize("mem_limit", [1, 10, 0]) def test_low_mem_limit_low_selectivity_scan(self, cursor, mem_limit, vector): """Tests that the queries specified in this test suite run under the given
