Repository: impala Updated Branches: refs/heads/2.x 7b8bd6a19 -> 22244bb07
IMPALA-7010: don't run memory usage tests on non-HDFS Moved a number of tests with tuned mem_limits. In some cases this required separating the tests from non-tuned functional tests. TestQueryMemLimit used very high and very low limits only, so seemed safe to run in all configurations. Change-Id: I9686195a29dde2d87b19ef8bb0e93e08f8bee662 Reviewed-on: http://gerrit.cloudera.org:8080/10370 Reviewed-by: Tim Armstrong <tarmstr...@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Reviewed-on: http://gerrit.cloudera.org:8080/10387 Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/22244bb0 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/22244bb0 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/22244bb0 Branch: refs/heads/2.x Commit: 22244bb0715417a2589cb53522debb14262bec06 Parents: 7b8bd6a Author: Tim Armstrong <tarmstr...@cloudera.com> Authored: Thu May 10 12:05:09 2018 -0700 Committer: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Committed: Sat May 12 06:29:51 2018 +0000 ---------------------------------------------------------------------- .../queries/QueryTest/insert-mem-limit.test | 16 ++ .../queries/QueryTest/insert.test | 16 +- .../queries/QueryTest/kudu_insert.test | 8 - .../QueryTest/kudu_insert_mem_limit.test | 10 + .../QueryTest/nested-types-tpch-mem-limit.test | 83 ++++++ .../queries/QueryTest/nested-types-tpch.test | 76 ------ .../queries/QueryTest/spilling-aggs.test | 40 --- .../spilling-regression-exhaustive.test | 268 +++++++++++++++++++ .../QueryTest/spilling-sorts-exhaustive.test | 224 ---------------- tests/common/skip.py | 5 +- tests/query_test/test_insert.py | 11 +- tests/query_test/test_kudu.py | 6 + tests/query_test/test_mem_usage_scaling.py | 10 +- tests/query_test/test_nested_types.py | 11 +- tests/query_test/test_sort.py | 4 +- tests/query_test/test_spilling.py | 7 +- 16 files changed, 418 insertions(+), 377 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/testdata/workloads/functional-query/queries/QueryTest/insert-mem-limit.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/insert-mem-limit.test b/testdata/workloads/functional-query/queries/QueryTest/insert-mem-limit.test new file mode 100644 index 0000000..84c1709 --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/insert-mem-limit.test @@ -0,0 +1,16 @@ +==== +---- QUERY +# Check that hdfs writers respects mem_limit. mem_limit is tuned for a 3-node HDFS +# minicluster. +set mem_limit=64m; +insert into table alltypesinsert +partition (year, month) /* +noclustered */ +select at1.id, at1.bool_col, at1.tinyint_col, at1.smallint_col, at1.int_col, at1.bigint_col, + at1.float_col, at1.double_col, at1.date_string_col, at1.string_col, at1.timestamp_col, + at1.year, at2.id as month +from functional.alltypes at1, functional.alltypes at2; +---- SETUP +DROP PARTITIONS alltypesinsert +---- CATCH +Memory limit exceeded +==== http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/testdata/workloads/functional-query/queries/QueryTest/insert.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/insert.test b/testdata/workloads/functional-query/queries/QueryTest/insert.test index 07a665a..0841151 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/insert.test +++ b/testdata/workloads/functional-query/queries/QueryTest/insert.test @@ -845,20 +845,6 @@ where year=2009 and month>=1 and month<=4 bigint ==== ---- QUERY -# Check that hdfs writers respects mem_limit. -set mem_limit=64m; -insert into table alltypesinsert -partition (year, month) -select at1.id, at1.bool_col, at1.tinyint_col, at1.smallint_col, at1.int_col, at1.bigint_col, - at1.float_col, at1.double_col, at1.date_string_col, at1.string_col, at1.timestamp_col, - at1.year, at2.id as month -from functional.alltypes at1, functional.alltypes at2; ----- SETUP -DROP PARTITIONS alltypesinsert ----- CATCH -Memory limit exceeded -==== ----- QUERY # IMPALA-2521: clustered insert into table insert into table alltypesinsert partition (year, month) /*+ clustered,shuffle */ @@ -955,4 +941,4 @@ right outer join functional.alltypestiny t1 on t1.id = v.id where v.id = 0 ---- RESULTS year=0/month=1/: 1 -==== \ No newline at end of file +==== http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test b/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test index da1bfac..75c61e7 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test +++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test @@ -451,11 +451,3 @@ insert into multiple_partition_cols select bigint_col, null, string_col NumModifiedRows: 0 NumRowErrors: 1 ==== ----- QUERY -# IMPALA-5611 - test a large insert with a memory limit -set mem_limit=400m; -create table kudu_test primary key(a, b) partition by hash(a, b) partitions 8 stored as kudu as -select l_orderkey a, concat(l_comment, l_comment, l_comment) b from tpch.lineitem ----- RUNTIME_PROFILE -row_regex: .*SortType: Partial -==== http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/testdata/workloads/functional-query/queries/QueryTest/kudu_insert_mem_limit.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu_insert_mem_limit.test b/testdata/workloads/functional-query/queries/QueryTest/kudu_insert_mem_limit.test new file mode 100644 index 0000000..b48c5ad --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_insert_mem_limit.test @@ -0,0 +1,10 @@ +==== +---- QUERY +# IMPALA-5611 - test a large insert with a memory limit. mem_limit is tuned for a 3-node +# HDFS minicluster. +set mem_limit=400m; +create table kudu_test primary key(a, b) partition by hash(a, b) partitions 8 stored as kudu as +select l_orderkey a, concat(l_comment, l_comment, l_comment) b from tpch.lineitem +---- RUNTIME_PROFILE +row_regex: .*SortType: Partial +==== http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch-mem-limit.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch-mem-limit.test b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch-mem-limit.test new file mode 100644 index 0000000..5591801 --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch-mem-limit.test @@ -0,0 +1,83 @@ +==== +---- QUERY +use tpch_nested_parquet +==== +---- QUERY +# IMPALA-2376: run scan that constructs large collection and set memory limit low enough +# to get the below query to consistently fail when allocating a large collection. Set +# num_nodes and mt_dop to 1 in order to make the query as deterministic as possible. +# mem_limit is tuned for a 3-node HDFS minicluster. +set buffer_pool_limit=24m; +set mem_limit=35m; +set num_nodes=1; +set mt_dop=1; +select max(cnt1), max(cnt2), max(cnt3), max(cnt4), max(cnt5) +from customer c, + (select count(l_returnflag) cnt1, count(l_partkey) cnt2, count(l_suppkey) cnt3, + count(l_linenumber) cnt4, count(l_quantity) cnt5 + from c.c_orders.o_lineitems) v; +---- TYPES +BIGINT +---- CATCH +row_regex: .*Memory limit exceeded: Failed to allocate [0-9]+ bytes for collection 'tpch_nested_parquet.customer.c_orders.item.o_lineitems'.* +==== +---- QUERY +# IMPALA-2473: Scan query with large row size leading to oversized batches. +# Should not OOM given single scanner thread and sufficient memory limit. +# mem_limit is tuned for a 3-node HDFS minicluster. +set num_scanner_threads=1; +set mem_limit=1g; +select * +from tpch_nested_parquet.customer c, c.c_orders o, o.o_lineitems +order by l_partkey desc, l_suppkey desc, l_linenumber desc, c_custkey +limit 5 +---- RESULTS +29888,'Customer#000029888','b8rX5B9YSZSnyro2o',6,'16-601-969-5769',8896.68,'FURNITURE','requests unwind carefully. slyly regular deposits across the fluffily final pac',3386277,'F',196816.57,'1993-11-29','2-HIGH','Clerk#000000176',0,'ular foxes cajole blithely slyly b',200000,7558,2,20.00,22000.00,0.03,0.07,'A','F','1994-01-13','1994-01-12','1994-02-02','COLLECT COD','REG AIR','ly express sauternes breach slyly' +51016,'Customer#000051016','DkC0qs6omn',23,'33-214-457-4366',5249.55,'MACHINERY','ual deposits according to the permanently regular instructions d',1500291,'F',252068.24,'1993-08-05','2-HIGH','Clerk#000000569',0,' are along the furiously even asymptotes. carefully ironic foxes accordin',200000,7558,2,44.00,48400.00,0.04,0.04,'R','F','1993-11-21','1993-10-26','1993-11-25','DELIVER IN PERSON','AIR','xpress packages. slyly regular theodoli' +90343,'Customer#000090343','KaC1qsGLktDWrWRuK4 Nrgbpvhs2rcg',4,'14-925-398-1742',4718.83,'MACHINERY',' even orbits. daringly pending accounts poach above the furiously even frays. final packages run. carefully stealthy',4315909,'O',221276.50,'1997-02-18','4-NOT SPECIFIED','Clerk#000000517',0,'lites: slyly final instructions integrate slyly pending, fina',200000,7558,1,32.00,35200.00,0.00,0.06,'N','O','1997-06-03','1997-04-11','1997-06-17','COLLECT COD','AIR','ns: special ' +111239,'Customer#000111239','xXeQMLtOCiS76XiMsRGzNdWCgZwsK S1l',6,'16-890-971-8054',3103.05,'FURNITURE','ites wake slyly according to the e',4186720,'F',277201.14,'1993-01-26','3-MEDIUM','Clerk#000000743',0,' regular ideas. stealthily bold accounts boost fu',200000,5039,3,47.00,51700.00,0.08,0.04,'A','F','1993-02-01','1993-03-26','1993-02-17','TAKE BACK RETURN','AIR','ias. slyly regular excuses are aft' +116908,'Customer#000116908','kCt00BJVwUSKxuOySsLlLCS',21,'31-857-393-5714',8079.23,'FURNITURE','ending platelets. final dependencies haggle about the ironic, express courts. s',3911265,'O',166014.13,'1997-07-27','3-MEDIUM','Clerk#000000161',0,'ily along the blithely ironic foxes. ',200000,5039,3,36.00,39600.00,0.10,0.02,'N','O','1997-10-31','1997-10-12','1997-11-12','TAKE BACK RETURN','REG AIR','y slyly special deposits. bli' +---- TYPES +bigint,string,string,smallint,string,decimal,string,string,bigint,string,decimal,string,string,string,int,string,bigint,bigint,int,decimal,decimal,decimal,decimal,string,string,string,string,string,string,string,string +==== +---- QUERY +# IMPALA-2473: Highly selective scan query with large row size. +# Should not OOM given single scanner thread and sufficient memory limit. +# mem_limit is tuned for a 3-node HDFS minicluster. +set num_scanner_threads=1; +set mem_limit=500m; +select * +from tpch_nested_parquet.customer c, c.c_orders o, o.o_lineitems +where c_phone='20-968-632-1388' and l_partkey = 127499 +---- RESULTS +53722,'Customer#000053722','p2hQ7009dQSEuRdE31HtjVv9idN2rXBHdTx4JRk',10,'20-968-632-1388',4153.75,'HOUSEHOLD','l foxes are ideas. final deposits boost blithely. carefully bo',10277,'F',101242.81,'1994-04-08','3-MEDIUM','Clerk#000000655',0,'counts sleep around the special',127499,2524,1,12.00,18317.88,0.07,0.07,'A','F','1994-07-27','1994-05-22','1994-08-11','NONE','RAIL',' according to the furiously silent th' +---- TYPES +bigint,string,string,smallint,string,decimal,string,string,bigint,string,decimal,string,string,string,int,string,bigint,bigint,int,decimal,decimal,decimal,decimal,string,string,string,string,string,string,string,string +==== +---- QUERY +# This was originally a regression test for IMPALA-5446: dropped status from +# Sorter::Reset() when sort cannot get reserved buffer. However with the +# IMPALA-3200 changes it now succeeds. +set buffer_pool_limit=180m; +select c_custkey, c_name, c_address, c_phone, c_acctbal, c_mktsegment, c_comment, + o_orderdate, sum(o_totalprice), min(rnum) +from customer c, + (select o_orderkey, o_totalprice, o_orderdate, row_number() over (order by o_orderdate desc) rnum + from c.c_orders) v +group by 1, 2, 3, 4, 5, 6, 7, 8 +order by 9, 10 desc +limit 10 +---- RESULTS +3115,'Customer#000003115','oB 75yHls7ptt5zCheWJLQ','22-291-864-7521',8889.56,'BUILDING','ts are quickly across the bold deposits. carefully spe','1998-04-23',857.71,3 +53551,'Customer#000053551','e,fT3URuJDH,tE6a6Z3Pjg0DZMFSqWbtYgd','15-429-275-5686',1137.38,'FURNITURE',' detect evenly along the blithely pending asymptotes. furiously even notornis detect carefu','1992-04-18',866.90,25 +64043,'Customer#000064043','Snyi GOB00','22-446-332-2750',4627.24,'FURNITURE','the quickly express asymptotes are around the pe','1992-01-31',870.88,11 +107698,'Customer#000107698','stUoykCwpTBAO3OC3lw','33-686-199-1188',698.89,'AUTOMOBILE',' accounts eat carefully express packages. slyly even id','1993-11-21',875.52,15 +1351,'Customer#000001351','NYMFfkNlCGoTeaDrNO9nn','11-916-210-6616',3106.00,'FURNITURE',' accounts after the final deposits sleep fluffily ironic accoun','1994-01-14',877.30,13 +85468,'Customer#000085468','EuFCX4qk4k0O4bV3UHoNVBTP','23-876-106-3120',8926.31,'AUTOMOBILE','kages. slyly even requests according to the ironic, ironic accounts cajole furiou','1997-04-12',884.52,4 +148522,'Customer#000148522','PIDMm8ulW4oam3VsoZL4f ,dpAf3LEV','16-597-824-4946',-133.27,'BUILDING','ly quickly express deposits. regularly regular requests cajole carefully slyly even noto','1995-03-20',885.75,12 +83222,'Customer#000083222','vI3tUuqtUYGPfrXAYeonVD9','27-599-263-5978',289.66,'BUILDING','ost quietly idle foxes. packages at the slyly pending pa','1993-05-02',891.74,5 +25090,'Customer#000025090','92GyVjZZiCBUmn','23-396-651-8663',8497.56,'BUILDING','osits. slyly final pinto beans sleep carefully fluffily express deposits. packages affix. carefully spe','1995-08-12',895.39,15 +27490,'Customer#000027490','jRzZQ1z7T,nrX5F58P,ZH','26-121-240-6744',7512.30,'AUTOMOBILE','slyly quickly even pinto beans: pend','1995-07-25',896.59,14 +---- TYPES +bigint,string,string,string,decimal,string,string,string,decimal,bigint +==== http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test index 9dae6e6..6a5166f 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test +++ b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test @@ -160,55 +160,6 @@ ORDER BY o_orderkey LIMIT 1 BIGINT,BIGINT ==== ---- QUERY -# IMPALA-2376: run scan that constructs large collection and set memory limit low enough -# to get the below query to consistently fail when allocating a large collection. Set -# num_nodes and mt_dop to 1 in order to make the query as deterministic as possible. -set buffer_pool_limit=24m; -set mem_limit=35m; -set num_nodes=1; -set mt_dop=1; -select max(cnt1), max(cnt2), max(cnt3), max(cnt4), max(cnt5) -from customer c, - (select count(l_returnflag) cnt1, count(l_partkey) cnt2, count(l_suppkey) cnt3, - count(l_linenumber) cnt4, count(l_quantity) cnt5 - from c.c_orders.o_lineitems) v; ----- TYPES -BIGINT ----- CATCH -row_regex: .*Memory limit exceeded: Failed to allocate [0-9]+ bytes for collection 'tpch_nested_parquet.customer.c_orders.item.o_lineitems'.* -==== ----- QUERY -# IMPALA-2473: Scan query with large row size leading to oversized batches. -# Should not OOM given single scanner thread and sufficient memory limit. -set num_scanner_threads=1; -set mem_limit=1g; -select * -from tpch_nested_parquet.customer c, c.c_orders o, o.o_lineitems -order by l_partkey desc, l_suppkey desc, l_linenumber desc, c_custkey -limit 5 ----- RESULTS -29888,'Customer#000029888','b8rX5B9YSZSnyro2o',6,'16-601-969-5769',8896.68,'FURNITURE','requests unwind carefully. slyly regular deposits across the fluffily final pac',3386277,'F',196816.57,'1993-11-29','2-HIGH','Clerk#000000176',0,'ular foxes cajole blithely slyly b',200000,7558,2,20.00,22000.00,0.03,0.07,'A','F','1994-01-13','1994-01-12','1994-02-02','COLLECT COD','REG AIR','ly express sauternes breach slyly' -51016,'Customer#000051016','DkC0qs6omn',23,'33-214-457-4366',5249.55,'MACHINERY','ual deposits according to the permanently regular instructions d',1500291,'F',252068.24,'1993-08-05','2-HIGH','Clerk#000000569',0,' are along the furiously even asymptotes. carefully ironic foxes accordin',200000,7558,2,44.00,48400.00,0.04,0.04,'R','F','1993-11-21','1993-10-26','1993-11-25','DELIVER IN PERSON','AIR','xpress packages. slyly regular theodoli' -90343,'Customer#000090343','KaC1qsGLktDWrWRuK4 Nrgbpvhs2rcg',4,'14-925-398-1742',4718.83,'MACHINERY',' even orbits. daringly pending accounts poach above the furiously even frays. final packages run. carefully stealthy',4315909,'O',221276.50,'1997-02-18','4-NOT SPECIFIED','Clerk#000000517',0,'lites: slyly final instructions integrate slyly pending, fina',200000,7558,1,32.00,35200.00,0.00,0.06,'N','O','1997-06-03','1997-04-11','1997-06-17','COLLECT COD','AIR','ns: special ' -111239,'Customer#000111239','xXeQMLtOCiS76XiMsRGzNdWCgZwsK S1l',6,'16-890-971-8054',3103.05,'FURNITURE','ites wake slyly according to the e',4186720,'F',277201.14,'1993-01-26','3-MEDIUM','Clerk#000000743',0,' regular ideas. stealthily bold accounts boost fu',200000,5039,3,47.00,51700.00,0.08,0.04,'A','F','1993-02-01','1993-03-26','1993-02-17','TAKE BACK RETURN','AIR','ias. slyly regular excuses are aft' -116908,'Customer#000116908','kCt00BJVwUSKxuOySsLlLCS',21,'31-857-393-5714',8079.23,'FURNITURE','ending platelets. final dependencies haggle about the ironic, express courts. s',3911265,'O',166014.13,'1997-07-27','3-MEDIUM','Clerk#000000161',0,'ily along the blithely ironic foxes. ',200000,5039,3,36.00,39600.00,0.10,0.02,'N','O','1997-10-31','1997-10-12','1997-11-12','TAKE BACK RETURN','REG AIR','y slyly special deposits. bli' ----- TYPES -bigint,string,string,smallint,string,decimal,string,string,bigint,string,decimal,string,string,string,int,string,bigint,bigint,int,decimal,decimal,decimal,decimal,string,string,string,string,string,string,string,string -==== ----- QUERY -# IMPALA-2473: Highly selective scan query with large row size. -# Should not OOM given single scanner thread and sufficient memory limit. -set num_scanner_threads=1; -set mem_limit=500m; -select * -from tpch_nested_parquet.customer c, c.c_orders o, o.o_lineitems -where c_phone='20-968-632-1388' and l_partkey = 127499 ----- RESULTS -53722,'Customer#000053722','p2hQ7009dQSEuRdE31HtjVv9idN2rXBHdTx4JRk',10,'20-968-632-1388',4153.75,'HOUSEHOLD','l foxes are ideas. final deposits boost blithely. carefully bo',10277,'F',101242.81,'1994-04-08','3-MEDIUM','Clerk#000000655',0,'counts sleep around the special',127499,2524,1,12.00,18317.88,0.07,0.07,'A','F','1994-07-27','1994-05-22','1994-08-11','NONE','RAIL',' according to the furiously silent th' ----- TYPES -bigint,string,string,smallint,string,decimal,string,string,bigint,string,decimal,string,string,string,int,string,bigint,bigint,int,decimal,decimal,decimal,decimal,string,string,string,string,string,string,string,string -==== ----- QUERY # IMPALA-4049: non-grouping aggregation with selective predicate in subplan feeding into # build side of a nested loop join. Reproduces a memory transfer bug triggered by empty # row batches in the build side of the join. @@ -238,30 +189,3 @@ order by c_custkey ---- TYPES bigint, bigint ==== ----- QUERY -# This was originally a regression test for IMPALA-5446: dropped status from -# Sorter::Reset() when sort cannot get reserved buffer. However with the -# IMPALA-3200 changes it now succeeds. -set buffer_pool_limit=180m; -select c_custkey, c_name, c_address, c_phone, c_acctbal, c_mktsegment, c_comment, - o_orderdate, sum(o_totalprice), min(rnum) -from customer c, - (select o_orderkey, o_totalprice, o_orderdate, row_number() over (order by o_orderdate desc) rnum - from c.c_orders) v -group by 1, 2, 3, 4, 5, 6, 7, 8 -order by 9, 10 desc -limit 10 ----- RESULTS -3115,'Customer#000003115','oB 75yHls7ptt5zCheWJLQ','22-291-864-7521',8889.56,'BUILDING','ts are quickly across the bold deposits. carefully spe','1998-04-23',857.71,3 -53551,'Customer#000053551','e,fT3URuJDH,tE6a6Z3Pjg0DZMFSqWbtYgd','15-429-275-5686',1137.38,'FURNITURE',' detect evenly along the blithely pending asymptotes. furiously even notornis detect carefu','1992-04-18',866.90,25 -64043,'Customer#000064043','Snyi GOB00','22-446-332-2750',4627.24,'FURNITURE','the quickly express asymptotes are around the pe','1992-01-31',870.88,11 -107698,'Customer#000107698','stUoykCwpTBAO3OC3lw','33-686-199-1188',698.89,'AUTOMOBILE',' accounts eat carefully express packages. slyly even id','1993-11-21',875.52,15 -1351,'Customer#000001351','NYMFfkNlCGoTeaDrNO9nn','11-916-210-6616',3106.00,'FURNITURE',' accounts after the final deposits sleep fluffily ironic accoun','1994-01-14',877.30,13 -85468,'Customer#000085468','EuFCX4qk4k0O4bV3UHoNVBTP','23-876-106-3120',8926.31,'AUTOMOBILE','kages. slyly even requests according to the ironic, ironic accounts cajole furiou','1997-04-12',884.52,4 -148522,'Customer#000148522','PIDMm8ulW4oam3VsoZL4f ,dpAf3LEV','16-597-824-4946',-133.27,'BUILDING','ly quickly express deposits. regularly regular requests cajole carefully slyly even noto','1995-03-20',885.75,12 -83222,'Customer#000083222','vI3tUuqtUYGPfrXAYeonVD9','27-599-263-5978',289.66,'BUILDING','ost quietly idle foxes. packages at the slyly pending pa','1993-05-02',891.74,5 -25090,'Customer#000025090','92GyVjZZiCBUmn','23-396-651-8663',8497.56,'BUILDING','osits. slyly final pinto beans sleep carefully fluffily express deposits. packages affix. carefully spe','1995-08-12',895.39,15 -27490,'Customer#000027490','jRzZQ1z7T,nrX5F58P,ZH','26-121-240-6744',7512.30,'AUTOMOBILE','slyly quickly even pinto beans: pend','1995-07-25',896.59,14 ----- TYPES -bigint,string,string,string,decimal,string,string,string,decimal,bigint -==== http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test index 20a03a1..fc53f8b 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test +++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test @@ -145,25 +145,6 @@ BIGINT, BIGINT, STRING row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\) ==== ---- QUERY -# Regression test for IMPALA-2612. The following query will cause CastToChar -# to be invoked when building the hash tables in partitioned aggregation -# nodes. CastToChar will do "local" memory allocation. Without the fix of -# IMPALA-2612, the peak memory consumption will be higher. -set mem_limit=800m; -set num_scanner_threads=1; -select count(distinct concat(cast(l_comment as char(120)), cast(l_comment as char(120)), - cast(l_comment as char(120)), cast(l_comment as char(120)), - cast(l_comment as char(120)), cast(l_comment as char(120)))) -from lineitem ----- RESULTS -4502054 ----- TYPES -BIGINT ----- RUNTIME_PROFILE -# Verify that the agg spilled. -row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\) -==== ----- QUERY # Test aggregation with minimum required reservation to exercise IMPALA-2708. # Merge aggregation requires 17 buffers and preaggregation requires 16 buffers # plus 1mb of hash tables. The buffer size is 256k for this test. The scan needs 24MB. @@ -178,24 +159,3 @@ BIGINT ---- RUNTIME_PROFILE row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\) ==== ----- QUERY -# IMPALA-3304: test that avg() can spill with a query mem limit. -# This test only covers that use FIXED_UDA_INTERMEDIATE, not functions that allocate -# strings for intermediate values. -set mem_limit=100m; -select l_orderkey, avg(l_tax), avg(l_quantity), avg(l_discount), avg(l_extendedprice) -from tpch_parquet.lineitem -group by 1 -order by avg(l_extendedprice) desc, avg(l_discount) -limit 5 ----- TYPES -BIGINT,DECIMAL,DECIMAL,DECIMAL,DECIMAL ----- RESULTS -3811460,0.05,50.00,0.05,104899.50 -1744195,0.04,50.00,0.09,104649.50 -5151266,0.07,50.00,0.00,104449.50 -4571042,0.03,50.00,0.09,104399.50 -1198304,0.01,50.00,0.02,104299.50 ----- RUNTIME_PROFILE -row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\) -==== http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/testdata/workloads/functional-query/queries/QueryTest/spilling-regression-exhaustive.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-regression-exhaustive.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-regression-exhaustive.test new file mode 100644 index 0000000..11c1aac --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-regression-exhaustive.test @@ -0,0 +1,268 @@ +==== +---- QUERY +# Test sort with small char column materialized by exprs. +# Set low mem_limit to force spilling. mem_limit is tuned for a 3-node HDFS minicluster. +# IMPALA-3332: comparator makes local allocations that cause runaway memory consumption. +set buffer_pool_limit=28m; +set mem_limit=200m; +set disable_outermost_topn=1; +select cast(l_comment as char(50)) +from lineitem +order by 1 +limit 20; +---- RESULTS +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias about the en ' +' Tiresias about the slyly ironic dinos ca ' +' Tiresias about the slyly unus ' +' Tiresias above ' +' Tiresias above the fox ' +' Tiresias above the furiously final th ' +' Tiresias above the slyly expr ' +' Tiresias above the stealthily p ' +---- TYPES +CHAR +---- RUNTIME_PROFILE +# Verify that the sort actually spilled +row_regex: .*SpilledRuns: .* \([1-9][0-9]*\) +row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\) +==== +---- QUERY +# Test sort with small input char column materialized before sort. +# mem_limit is tuned for a 3-node HDFS minicluster. +set mem_limit=200m; +set buffer_pool_limit=28m; +set disable_outermost_topn=1; +select char_col +from (select cast(l_comment as char(50)) char_col + from lineitem) subquery +order by 1 +limit 20; +---- RESULTS +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias about the en ' +' Tiresias about the slyly ironic dinos ca ' +' Tiresias about the slyly unus ' +' Tiresias above ' +' Tiresias above the fox ' +' Tiresias above the furiously final th ' +' Tiresias above the slyly expr ' +' Tiresias above the stealthily p ' +---- TYPES +CHAR +---- RUNTIME_PROFILE +# Verify that the sort actually spilled +row_regex: .*SpilledRuns: .* \([1-9][0-9]*\) +row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\) +==== +---- QUERY +# Test sort with large input char column materialized before sort. +# Set low mem_limit to force spilling. mem_limit is tuned for a 3-node HDFS minicluster. +set mem_limit=200m; +set buffer_pool_limit=28m; +set disable_outermost_topn=1; +select char_col +from (select cast(l_comment as char(200)) char_col + from lineitem) subquery +order by 1 +limit 20; +---- RESULTS +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias about the en ' +' Tiresias about the slyly ironic dinos ca ' +' Tiresias about the slyly unus ' +' Tiresias above ' +' Tiresias above the fox ' +' Tiresias above the furiously final th ' +' Tiresias above the slyly expr ' +' Tiresias above the stealthily p ' +---- TYPES +CHAR +---- RUNTIME_PROFILE +# Verify that the sort actually spilled +row_regex: .*SpilledRuns: .* \([1-9][0-9]*\) +row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\) +==== +---- QUERY +# Test sort with varchar column materialized by exprs. +# Set low mem_limit to force spilling. mem_limit is tuned for a 3-node HDFS minicluster. +set buffer_pool_limit=28m; +# IMPALA-3332: comparator makes local allocations that cause runaway memory consumption. +set mem_limit=200m; +set disable_outermost_topn=1; +select cast(l_comment as varchar(50)) +from lineitem +order by 1 +limit 20; +---- RESULTS +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias about the en' +' Tiresias about the slyly ironic dinos ca' +' Tiresias about the slyly unus' +' Tiresias above' +' Tiresias above the fox' +' Tiresias above the furiously final th' +' Tiresias above the slyly expr' +' Tiresias above the stealthily p' +---- TYPES +STRING +---- RUNTIME_PROFILE +# Verify that the sort actually spilled +row_regex: .*SpilledRuns: .* \([1-9][0-9]*\) +row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\) +==== +---- QUERY +# Test sort with input varchar column materialized before sort. +# Set low mem_limit to force spilling. mem_limit is tuned for a 3-node HDFS minicluster. +set mem_limit=200m; +set buffer_pool_limit=28m; +set disable_outermost_topn=1; +select char_col +from (select cast(l_comment as varchar(50)) char_col + from lineitem) subquery +order by 1 +limit 20; +---- RESULTS +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias ' +' Tiresias about the en' +' Tiresias about the slyly ironic dinos ca' +' Tiresias about the slyly unus' +' Tiresias above' +' Tiresias above the fox' +' Tiresias above the furiously final th' +' Tiresias above the slyly expr' +' Tiresias above the stealthily p' +---- TYPES +STRING +---- RUNTIME_PROFILE +# Verify that the sort actually spilled +row_regex: .*SpilledRuns: .* \([1-9][0-9]*\) +row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\) +==== +---- QUERY +# Test that we can spill with a buffer size larger than the default I/O size (8mb). +# buffer_pool_limit is tuned for a 3-node HDFS minicluster. +set default_spillable_buffer_size=16m; +set buffer_pool_limit=96m; +select l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment +from tpch_parquet.lineitem +order by 1, 2, 3, 4, 5 +offset 6001200 +---- TYPES +BIGINT,BIGINT,BIGINT,INT,STRING +---- RESULTS +5999971,97922,7923,5,'place of the slyly quick pla' +5999971,132903,2904,4,'ckly above the boldly ' +5999971,144823,4824,2,'e blithely after the carefully pending ' +5999971,161882,6915,6,'luffy theodolites nag boldly bli' +5999972,102942,473,1,'s maintain carefully among the' +5999972,133109,8136,2,' the furiously express pearls. furi' +5999972,152761,2762,3,'sual accounts al' +5999973,176345,1380,1,'gular excuses. ' +5999974,10463,5466,2,'dolites wake' +5999974,25360,5361,1,'express dependencies. express, pendi' +5999975,6452,1453,2,'lar pinto beans aft' +5999975,7272,2273,1,'tructions. excu' +5999975,37131,2138,3,', quick deposits. ironic, unusual deposi' +6000000,32255,2256,1,'carefully ' +6000000,96127,6128,2,'ooze furiously about the pe' +---- RUNTIME_PROFILE +row_regex: .*SpilledRuns: .* \([1-9][0-9]*\) +==== +---- QUERY +# Regression test for IMPALA-2612. The following query will cause CastToChar +# to be invoked when building the hash tables in partitioned aggregation +# nodes. CastToChar will do "local" memory allocation. Without the fix of +# IMPALA-2612, the peak memory consumption will be higher. +# mem_limit is tuned to reproduce the issue on a 3-node HDFS minicluster. +set mem_limit=800m; +set num_scanner_threads=1; +select count(distinct concat(cast(l_comment as char(120)), cast(l_comment as char(120)), + cast(l_comment as char(120)), cast(l_comment as char(120)), + cast(l_comment as char(120)), cast(l_comment as char(120)))) +from lineitem +---- RESULTS +4502054 +---- TYPES +BIGINT +---- RUNTIME_PROFILE +# Verify that the agg spilled. +row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\) +==== +---- QUERY +# IMPALA-3304: test that avg() can spill with a query mem limit. +# This test only covers that use FIXED_UDA_INTERMEDIATE, not functions that allocate +# strings for intermediate values. mem_limit is tuned to reproduce the issue on a 3-node +# HDFS minicluster. +set mem_limit=100m; +select l_orderkey, avg(l_tax), avg(l_quantity), avg(l_discount), avg(l_extendedprice) +from tpch_parquet.lineitem +group by 1 +order by avg(l_extendedprice) desc, avg(l_discount) +limit 5 +---- TYPES +BIGINT,DECIMAL,DECIMAL,DECIMAL,DECIMAL +---- RESULTS +3811460,0.05,50.00,0.05,104899.50 +1744195,0.04,50.00,0.09,104649.50 +5151266,0.07,50.00,0.00,104449.50 +4571042,0.03,50.00,0.09,104399.50 +1198304,0.01,50.00,0.02,104299.50 +---- RUNTIME_PROFILE +row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\) +==== http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test deleted file mode 100644 index 7d65523..0000000 --- a/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test +++ /dev/null @@ -1,224 +0,0 @@ -==== ----- QUERY -# Test sort with small char column materialized by exprs. -# Set low memory limit to force spilling. -# IMPALA-3332: comparator makes local allocations that cause runaway memory consumption. -set buffer_pool_limit=28m; -set mem_limit=200m; -set disable_outermost_topn=1; -select cast(l_comment as char(50)) -from lineitem -order by 1 -limit 20; ----- RESULTS -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias about the en ' -' Tiresias about the slyly ironic dinos ca ' -' Tiresias about the slyly unus ' -' Tiresias above ' -' Tiresias above the fox ' -' Tiresias above the furiously final th ' -' Tiresias above the slyly expr ' -' Tiresias above the stealthily p ' ----- TYPES -CHAR ----- RUNTIME_PROFILE -# Verify that the sort actually spilled -row_regex: .*SpilledRuns: .* \([1-9][0-9]*\) -row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\) -==== ----- QUERY -# Test sort with small input char column materialized before sort. -set mem_limit=200m; -set buffer_pool_limit=28m; -set disable_outermost_topn=1; -select char_col -from (select cast(l_comment as char(50)) char_col - from lineitem) subquery -order by 1 -limit 20; ----- RESULTS -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias about the en ' -' Tiresias about the slyly ironic dinos ca ' -' Tiresias about the slyly unus ' -' Tiresias above ' -' Tiresias above the fox ' -' Tiresias above the furiously final th ' -' Tiresias above the slyly expr ' -' Tiresias above the stealthily p ' ----- TYPES -CHAR ----- RUNTIME_PROFILE -# Verify that the sort actually spilled -row_regex: .*SpilledRuns: .* \([1-9][0-9]*\) -row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\) -==== ----- QUERY -# Test sort with large input char column materialized before sort. -# Set low memory limit to force spilling. -set mem_limit=200m; -set buffer_pool_limit=28m; -set disable_outermost_topn=1; -select char_col -from (select cast(l_comment as char(200)) char_col - from lineitem) subquery -order by 1 -limit 20; ----- RESULTS -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias about the en ' -' Tiresias about the slyly ironic dinos ca ' -' Tiresias about the slyly unus ' -' Tiresias above ' -' Tiresias above the fox ' -' Tiresias above the furiously final th ' -' Tiresias above the slyly expr ' -' Tiresias above the stealthily p ' ----- TYPES -CHAR ----- RUNTIME_PROFILE -# Verify that the sort actually spilled -row_regex: .*SpilledRuns: .* \([1-9][0-9]*\) -row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\) -==== ----- QUERY -# Test sort with varchar column materialized by exprs. -# Set low memory limit to force spilling. -set buffer_pool_limit=28m; -# IMPALA-3332: comparator makes local allocations that cause runaway memory consumption. -set mem_limit=200m; -set disable_outermost_topn=1; -select cast(l_comment as varchar(50)) -from lineitem -order by 1 -limit 20; ----- RESULTS -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias about the en' -' Tiresias about the slyly ironic dinos ca' -' Tiresias about the slyly unus' -' Tiresias above' -' Tiresias above the fox' -' Tiresias above the furiously final th' -' Tiresias above the slyly expr' -' Tiresias above the stealthily p' ----- TYPES -STRING ----- RUNTIME_PROFILE -# Verify that the sort actually spilled -row_regex: .*SpilledRuns: .* \([1-9][0-9]*\) -row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\) -==== ----- QUERY -# Test sort with input varchar column materialized before sort. -# Set low memory limit to force spilling. -set mem_limit=200m; -set buffer_pool_limit=28m; -set disable_outermost_topn=1; -select char_col -from (select cast(l_comment as varchar(50)) char_col - from lineitem) subquery -order by 1 -limit 20; ----- RESULTS -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias ' -' Tiresias about the en' -' Tiresias about the slyly ironic dinos ca' -' Tiresias about the slyly unus' -' Tiresias above' -' Tiresias above the fox' -' Tiresias above the furiously final th' -' Tiresias above the slyly expr' -' Tiresias above the stealthily p' ----- TYPES -STRING ----- RUNTIME_PROFILE -# Verify that the sort actually spilled -row_regex: .*SpilledRuns: .* \([1-9][0-9]*\) -row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\) -==== ----- QUERY -# Test that we can spill with a buffer size larger than the default I/O size (8mb). -set default_spillable_buffer_size=16m; -set buffer_pool_limit=96m; -select l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment -from tpch_parquet.lineitem -order by 1, 2, 3, 4, 5 -offset 6001200 ----- TYPES -BIGINT,BIGINT,BIGINT,INT,STRING ----- RESULTS -5999971,97922,7923,5,'place of the slyly quick pla' -5999971,132903,2904,4,'ckly above the boldly ' -5999971,144823,4824,2,'e blithely after the carefully pending ' -5999971,161882,6915,6,'luffy theodolites nag boldly bli' -5999972,102942,473,1,'s maintain carefully among the' -5999972,133109,8136,2,' the furiously express pearls. furi' -5999972,152761,2762,3,'sual accounts al' -5999973,176345,1380,1,'gular excuses. ' -5999974,10463,5466,2,'dolites wake' -5999974,25360,5361,1,'express dependencies. express, pendi' -5999975,6452,1453,2,'lar pinto beans aft' -5999975,7272,2273,1,'tructions. excu' -5999975,37131,2138,3,', quick deposits. ironic, unusual deposi' -6000000,32255,2256,1,'carefully ' -6000000,96127,6128,2,'ooze furiously about the pe' ----- RUNTIME_PROFILE -row_regex: .*SpilledRuns: .* \([1-9][0-9]*\) -==== http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/tests/common/skip.py ---------------------------------------------------------------------- diff --git a/tests/common/skip.py b/tests/common/skip.py index 7d515f7..9a5d95d 100644 --- a/tests/common/skip.py +++ b/tests/common/skip.py @@ -127,8 +127,6 @@ class SkipIfLocal: reason="HBase not started when using local file system") hdfs_client = pytest.mark.skipif(IS_LOCAL, reason="HDFS not started when using local file system") - mem_usage_different = pytest.mark.skipif(IS_LOCAL, - reason="Memory limit too low when running single node") qualified_path = pytest.mark.skipif(IS_LOCAL, reason="Tests rely on HDFS qualified paths") root_path = pytest.mark.skipif(IS_LOCAL, @@ -138,6 +136,9 @@ class SkipIfNotHdfsMinicluster: # These ones are skipped when not running against a local HDFS mini-cluster. plans = pytest.mark.skipif(not IS_HDFS or pytest.config.option.testing_remote_cluster, reason="Test assumes plans from local HDFS mini-cluster") + tuned_for_minicluster = pytest.mark.skipif( + not IS_HDFS or pytest.config.option.testing_remote_cluster, + reason="Test is tuned for 3-node HDFS minicluster") class SkipIfBuildType: not_dev_build = pytest.mark.skipif(not IMPALAD_BUILD.is_dev(), http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/tests/query_test/test_insert.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_insert.py b/tests/query_test/test_insert.py index 7a1b935..2307b76 100644 --- a/tests/query_test/test_insert.py +++ b/tests/query_test/test_insert.py @@ -21,7 +21,7 @@ import pytest from testdata.common import widetable from tests.common.impala_test_suite import ImpalaTestSuite -from tests.common.skip import SkipIfLocal +from tests.common.skip import SkipIfLocal, SkipIfNotHdfsMinicluster from tests.common.test_dimensions import ( create_exec_option_dimension, create_uncompressed_text_dimension) @@ -118,6 +118,15 @@ class TestInsertQueries(ImpalaTestSuite): multiple_impalad=vector.get_value('exec_option')['sync_ddl'] == 1) @pytest.mark.execute_serially + @SkipIfNotHdfsMinicluster.tuned_for_minicluster + def test_insert_mem_limit(self, vector): + if (vector.get_value('table_format').file_format == 'parquet'): + vector.get_value('exec_option')['COMPRESSION_CODEC'] = \ + vector.get_value('compression_codec') + self.run_test_case('QueryTest/insert-mem-limit', vector, + multiple_impalad=vector.get_value('exec_option')['sync_ddl'] == 1) + + @pytest.mark.execute_serially def test_insert_overwrite(self, vector): self.run_test_case('QueryTest/insert_overwrite', vector, multiple_impalad=vector.get_value('exec_option')['sync_ddl'] == 1) http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/tests/query_test/test_kudu.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py index ae76f75..d6a0d2a 100644 --- a/tests/query_test/test_kudu.py +++ b/tests/query_test/test_kudu.py @@ -39,6 +39,7 @@ from pytz import utc from tests.common.kudu_test_suite import KuduTestSuite from tests.common.impala_cluster import ImpalaCluster +from tests.common.skip import SkipIfNotHdfsMinicluster from tests.verifiers.metric_verifier import MetricVerifier KUDU_MASTER_HOSTS = pytest.config.option.kudu_master_hosts @@ -83,6 +84,10 @@ class TestKuduOperations(KuduTestSuite): def test_kudu_insert(self, vector, unique_database): self.run_test_case('QueryTest/kudu_insert', vector, use_db=unique_database) + @SkipIfNotHdfsMinicluster.tuned_for_minicluster + def test_kudu_insert_mem_limit(self, vector, unique_database): + self.run_test_case('QueryTest/kudu_insert_mem_limit', vector, use_db=unique_database) + def test_kudu_update(self, vector, unique_database): self.run_test_case('QueryTest/kudu_update', vector, use_db=unique_database) @@ -1028,6 +1033,7 @@ class TestImpalaKuduIntegration(KuduTestSuite): cursor.execute("SHOW TABLES IN %s" % unique_database) assert (impala_tbl_name,) not in cursor.fetchall() +@SkipIfNotHdfsMinicluster.tuned_for_minicluster class TestKuduMemLimits(KuduTestSuite): QUERIES = ["select * from tpch_kudu.lineitem where l_orderkey = -1", http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/tests/query_test/test_mem_usage_scaling.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_mem_usage_scaling.py b/tests/query_test/test_mem_usage_scaling.py index 749f77f..7fb0fda 100644 --- a/tests/query_test/test_mem_usage_scaling.py +++ b/tests/query_test/test_mem_usage_scaling.py @@ -21,7 +21,7 @@ from copy import copy from tests.beeswax.impala_beeswax import ImpalaBeeswaxException from tests.common.impala_cluster import ImpalaCluster from tests.common.impala_test_suite import ImpalaTestSuite -from tests.common.skip import SkipIfLocal +from tests.common.skip import SkipIfNotHdfsMinicluster from tests.common.test_dimensions import create_single_exec_option_dimension from tests.common.test_vector import ImpalaTestDimension from tests.verifiers.metric_verifier import MetricVerifier @@ -32,7 +32,7 @@ MEM_LIMIT_TOO_LOW_FOR_RESERVATION = ("minimum memory reservation is greater than "available to the query for buffer reservations") MEM_LIMIT_ERROR_MSGS = [MEM_LIMIT_EXCEEDED_MSG, MEM_LIMIT_TOO_LOW_FOR_RESERVATION] -@SkipIfLocal.mem_usage_different +@SkipIfNotHdfsMinicluster.tuned_for_minicluster class TestQueryMemLimitScaling(ImpalaTestSuite): """Test class to do functional validation of per query memory limits. """ QUERY = ["select * from lineitem where l_orderkey = -1", @@ -120,7 +120,7 @@ class TestLowMemoryLimits(ImpalaTestSuite): assert found_expected_error, str(e) -@SkipIfLocal.mem_usage_different +@SkipIfNotHdfsMinicluster.tuned_for_minicluster class TestTpchMemLimitError(TestLowMemoryLimits): # The mem limits that will be used. MEM_IN_MB = [20, 50, 80, 130, 160, 200, 400] @@ -226,7 +226,7 @@ class TestTpchMemLimitError(TestLowMemoryLimits): verifier = MetricVerifier(impalad.service) verifier.wait_for_metric("impala-server.num-fragments-in-flight", 0) -@SkipIfLocal.mem_usage_different +@SkipIfNotHdfsMinicluster.tuned_for_minicluster class TestTpchPrimitivesMemLimitError(TestLowMemoryLimits): """ Memory usage tests using targeted-perf queries to exercise specific operators. @@ -272,7 +272,7 @@ class TestTpchPrimitivesMemLimitError(TestLowMemoryLimits): self.run_primitive_query(vector, 'primitive_orderby_all') -@SkipIfLocal.mem_usage_different +@SkipIfNotHdfsMinicluster.tuned_for_minicluster class TestTpcdsMemLimitError(TestLowMemoryLimits): # The mem limits that will be used. MEM_IN_MB = [20, 100, 116, 150] http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/tests/query_test/test_nested_types.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_nested_types.py b/tests/query_test/test_nested_types.py index 85ed3a4..c83c06d 100644 --- a/tests/query_test/test_nested_types.py +++ b/tests/query_test/test_nested_types.py @@ -26,7 +26,9 @@ from tests.common.skip import ( SkipIfIsilon, SkipIfS3, SkipIfADLS, - SkipIfLocal) + SkipIfLocal, + SkipIfNotHdfsMinicluster) +from tests.common.test_vector import ImpalaTestDimension from tests.util.filesystem_utils import WAREHOUSE, get_fs_path class TestNestedTypes(ImpalaTestSuite): @@ -73,11 +75,16 @@ class TestNestedTypes(ImpalaTestSuite): """Queries using nested types and with WITH clause.""" self.run_test_case('QueryTest/nested-types-with-clause', vector) - @SkipIfLocal.mem_usage_different def test_tpch(self, vector): """Queries over the larger nested TPCH dataset.""" self.run_test_case('QueryTest/nested-types-tpch', vector) + @SkipIfNotHdfsMinicluster.tuned_for_minicluster + def test_tpch_mem_limit(self, vector): + """Queries over the larger nested TPCH dataset with memory limits tuned for + a 3-node HDFS minicluster.""" + self.run_test_case('QueryTest/nested-types-tpch-mem-limit', vector) + def test_parquet_stats(self, vector): """Queries that test evaluation of Parquet row group statistics.""" self.run_test_case('QueryTest/nested-types-parquet-stats', vector) http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/tests/query_test/test_sort.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_sort.py b/tests/query_test/test_sort.py index 379b758..a6ffd28 100644 --- a/tests/query_test/test_sort.py +++ b/tests/query_test/test_sort.py @@ -18,7 +18,7 @@ from copy import copy, deepcopy from tests.common.impala_test_suite import ImpalaTestSuite -from tests.common.skip import SkipIfLocal +from tests.common.skip import SkipIfNotHdfsMinicluster def transpose_results(result, map_fn=lambda x: x): """Given a query result (list of strings, each string represents a row), return a list @@ -168,7 +168,7 @@ class TestQueryFullSort(ImpalaTestSuite): query, exec_option, table_format=table_format).data) assert(result[0] == sorted(result[0])) - @SkipIfLocal.mem_usage_different + @SkipIfNotHdfsMinicluster.tuned_for_minicluster def test_sort_reservation_usage(self, vector): """Tests for sorter reservation usage.""" new_vector = deepcopy(vector) http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/tests/query_test/test_spilling.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_spilling.py b/tests/query_test/test_spilling.py index 8629853..0b40d78 100644 --- a/tests/query_test/test_spilling.py +++ b/tests/query_test/test_spilling.py @@ -18,6 +18,7 @@ import pytest from tests.common.impala_test_suite import ImpalaTestSuite +from tests.common.skip import SkipIfNotHdfsMinicluster from tests.common.test_dimensions import (create_exec_option_dimension_from_dict, create_parquet_dimension) @@ -69,10 +70,12 @@ class TestSpillingDebugActionDimensions(ImpalaTestSuite): """Test spilling null-aware anti-joins""" self.run_test_case('QueryTest/spilling-naaj', vector) - def test_spilling_sorts_exhaustive(self, vector): + @SkipIfNotHdfsMinicluster.tuned_for_minicluster + def test_spilling_regression_exhaustive(self, vector): + """Regression tests for spilling. mem_limits tuned for 3-node minicluster.""" if self.exploration_strategy() != 'exhaustive': pytest.skip("only run large sorts on exhaustive") - self.run_test_case('QueryTest/spilling-sorts-exhaustive', vector) + self.run_test_case('QueryTest/spilling-regression-exhaustive', vector) @pytest.mark.xfail(pytest.config.option.testing_remote_cluster,
