Repository: incubator-hawq Updated Branches: refs/heads/master 1470d5302 -> 68013fac1
HAWQ-1359. Fix bugs in policy tests for HAWQ with Ranger enabled and add test for HAWQ-1367 Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/68013fac Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/68013fac Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/68013fac Branch: refs/heads/master Commit: 68013fac13a757489d7a7ebe361065826ea5d66a Parents: 1470d53 Author: Chunling Wang <[email protected]> Authored: Fri Mar 3 11:23:32 2017 +0800 Committer: Chunling Wang <[email protected]> Committed: Fri Mar 3 11:23:32 2017 +0800 ---------------------------------------------------------------------- src/test/feature/Ranger/ans/adminfirst41.ans | 8 ++------ src/test/feature/Ranger/ans/adminsecond41.ans | 16 ++-------------- src/test/feature/Ranger/ans/normal35_success.ans | 2 +- src/test/feature/Ranger/ans/normal41_fail.ans | 8 ++++---- src/test/feature/Ranger/ans/normal41_success.ans | 8 ++------ src/test/feature/Ranger/ans/super63_fail.ans | 6 +++--- src/test/feature/Ranger/sql/admin/41.sql | 2 +- src/test/feature/Ranger/sql/init_file | 4 ++++ src/test/feature/Ranger/sql/normal/41.sql | 2 +- src/test/feature/Ranger/sql/sqlsplitinput.sql | 2 +- src/test/feature/Ranger/test_ranger.cpp | 12 ++++++------ src/test/feature/gtest-parallel | 13 +++++++------ 12 files changed, 34 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68013fac/src/test/feature/Ranger/ans/adminfirst41.ans ---------------------------------------------------------------------- diff --git a/src/test/feature/Ranger/ans/adminfirst41.ans b/src/test/feature/Ranger/ans/adminfirst41.ans index d28e95d..ce0b96d 100644 --- a/src/test/feature/Ranger/ans/adminfirst41.ans +++ b/src/test/feature/Ranger/ans/adminfirst41.ans @@ -1,8 +1,4 @@ -- start_ignore -- end_ignore -select * from a order by i; -1 -1 -1 -1 -1 +select * from pg_database, a order by oid, i limit 1; +template1|10|6|t|t|-1|0|881|1663|16385|||1 http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68013fac/src/test/feature/Ranger/ans/adminsecond41.ans ---------------------------------------------------------------------- diff --git a/src/test/feature/Ranger/ans/adminsecond41.ans b/src/test/feature/Ranger/ans/adminsecond41.ans index 2930484..ce0b96d 100644 --- a/src/test/feature/Ranger/ans/adminsecond41.ans +++ b/src/test/feature/Ranger/ans/adminsecond41.ans @@ -1,16 +1,4 @@ -- start_ignore -- end_ignore -select * from a order by i; -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -2 +select * from pg_database, a order by oid, i limit 1; +template1|10|6|t|t|-1|0|881|1663|16385|||1 http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68013fac/src/test/feature/Ranger/ans/normal35_success.ans ---------------------------------------------------------------------- diff --git a/src/test/feature/Ranger/ans/normal35_success.ans b/src/test/feature/Ranger/ans/normal35_success.ans index 0957278..cf37292 100644 --- a/src/test/feature/Ranger/ans/normal35_success.ans +++ b/src/test/feature/Ranger/ans/normal35_success.ans @@ -3,6 +3,6 @@ set session role=usertest35; SET create table aa as select * from a order by i; -psql:/tmp/TestHawqRanger_BasicTest.sql:4: WARNING: skipping "aa" --- error returned: permission denied for relation(s): aa +psql:/tmp/TestHawqRanger_BasicTest.sql:4: WARNING: skipping "aa" --- error returned: permission denied for relation(s): public.aa psql:/tmp/TestHawqRanger_BasicTest.sql:4: INFO: ANALYZE completed. Success: 0, Failure: 1 (aa) SELECT 2 http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68013fac/src/test/feature/Ranger/ans/normal41_fail.ans ---------------------------------------------------------------------- diff --git a/src/test/feature/Ranger/ans/normal41_fail.ans b/src/test/feature/Ranger/ans/normal41_fail.ans index 8ac9d96..095b94f 100644 --- a/src/test/feature/Ranger/ans/normal41_fail.ans +++ b/src/test/feature/Ranger/ans/normal41_fail.ans @@ -2,10 +2,10 @@ -- end_ignore set session role=usertest41; SET -select * from a order by i; +select * from pg_database, a order by oid, i limit 1; psql:/tmp/TestHawqRanger_BasicTest.sql:4: WARNING: usage privilege of namespace public is required. -LINE 1: select * from a order by i; +LINE 1: select * from pg_database, a order by oid, i limit 1; ^ psql:/tmp/TestHawqRanger_BasicTest.sql:4: ERROR: relation "a" does not exist -LINE 1: select * from a order by i; - ^ +LINE 1: select * from pg_database, a order by oid, i limit 1; + ^ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68013fac/src/test/feature/Ranger/ans/normal41_success.ans ---------------------------------------------------------------------- diff --git a/src/test/feature/Ranger/ans/normal41_success.ans b/src/test/feature/Ranger/ans/normal41_success.ans index a49b3f9..d20bfa6 100644 --- a/src/test/feature/Ranger/ans/normal41_success.ans +++ b/src/test/feature/Ranger/ans/normal41_success.ans @@ -2,9 +2,5 @@ -- end_ignore set session role=usertest41; SET -select * from a order by i; -1 -1 -1 -1 -1 +select * from pg_database, a order by oid, i limit 1; +template1|10|6|t|t|-1|0|881|1663|16385|||1 http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68013fac/src/test/feature/Ranger/ans/super63_fail.ans ---------------------------------------------------------------------- diff --git a/src/test/feature/Ranger/ans/super63_fail.ans b/src/test/feature/Ranger/ans/super63_fail.ans index 07b2259..6f708ca 100644 --- a/src/test/feature/Ranger/ans/super63_fail.ans +++ b/src/test/feature/Ranger/ans/super63_fail.ans @@ -3,8 +3,8 @@ set session role=usersuper63; SET vacuum analyze; -psql:/tmp/TestHawqRanger_BasicTest.sql:4: WARNING: skipping "a" --- error returned: permission denied for relation(s): a -psql:/tmp/TestHawqRanger_BasicTest.sql:4: WARNING: skipping "aa" --- error returned: permission denied for relation(s): aa -psql:/tmp/TestHawqRanger_BasicTest.sql:4: WARNING: skipping "aaa" --- error returned: permission denied for relation(s): aaa +psql:/tmp/TestHawqRanger_BasicTest.sql:4: WARNING: skipping "a" --- error returned: permission denied for relation(s): public.a +psql:/tmp/TestHawqRanger_BasicTest.sql:4: WARNING: skipping "aa" --- error returned: permission denied for relation(s): public.aa +psql:/tmp/TestHawqRanger_BasicTest.sql:4: WARNING: skipping "aaa" --- error returned: permission denied for relation(s): public.aaa psql:/tmp/TestHawqRanger_BasicTest.sql:4: INFO: ANALYZE completed. Success: 85, Failure: 3 (a, aa, aaa) VACUUM http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68013fac/src/test/feature/Ranger/sql/admin/41.sql ---------------------------------------------------------------------- diff --git a/src/test/feature/Ranger/sql/admin/41.sql b/src/test/feature/Ranger/sql/admin/41.sql index 385baa2..e658e5c 100644 --- a/src/test/feature/Ranger/sql/admin/41.sql +++ b/src/test/feature/Ranger/sql/admin/41.sql @@ -1,2 +1,2 @@ -select * from a order by i; +select * from pg_database, a order by oid, i limit 1; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68013fac/src/test/feature/Ranger/sql/init_file ---------------------------------------------------------------------- diff --git a/src/test/feature/Ranger/sql/init_file b/src/test/feature/Ranger/sql/init_file index 5f4c68d..6c5dbe7 100644 --- a/src/test/feature/Ranger/sql/init_file +++ b/src/test/feature/Ranger/sql/init_file @@ -14,8 +14,12 @@ # limitations under the License. -- start_matchignore m/You are now connected to database*/ +m/Settings: default_hash_table_bucket_number=.*/ +m/[O|o]ptimizer status:.*/ s/public\|a\|table\|.*\|append only/public\|a\|table\|gpadmin\|append only/ s/public\|aa\|table\|.*\|append only/public\|aa\|table\|gpadmin\|append only/ s/public\|aaa\|table\|.*\|append only/public\|aaa\|table\|gpadmin\|append only/ s/public\|av\|view\|.*\|none/public\|av\|view\|gpadmin\|none/ +s/public\|myseq\|sequence\|.*\|heap/public\|myseq\|sequence\|gpadmin\|heap/ +s/cost=.* rows/cost=fixedcost rows/ -- end_matchignore http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68013fac/src/test/feature/Ranger/sql/normal/41.sql ---------------------------------------------------------------------- diff --git a/src/test/feature/Ranger/sql/normal/41.sql b/src/test/feature/Ranger/sql/normal/41.sql index 57bb612..9db3444 100644 --- a/src/test/feature/Ranger/sql/normal/41.sql +++ b/src/test/feature/Ranger/sql/normal/41.sql @@ -1,3 +1,3 @@ set session role=usertest41; -select * from a order by i; +select * from pg_database, a order by oid, i limit 1; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68013fac/src/test/feature/Ranger/sql/sqlsplitinput.sql ---------------------------------------------------------------------- diff --git a/src/test/feature/Ranger/sql/sqlsplitinput.sql b/src/test/feature/Ranger/sql/sqlsplitinput.sql index 564b0a0..3b3d465 100644 --- a/src/test/feature/Ranger/sql/sqlsplitinput.sql +++ b/src/test/feature/Ranger/sql/sqlsplitinput.sql @@ -79,7 +79,7 @@ insert into a values(1); 1 insert into a VALUES (nextval('myseq')); 1 -select * from a order by i; +select * from pg_database, a order by oid, i limit 1; 1 select generate_series(1,3); 1 http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68013fac/src/test/feature/Ranger/test_ranger.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/Ranger/test_ranger.cpp b/src/test/feature/Ranger/test_ranger.cpp index 1d9f08e..acc8b97 100644 --- a/src/test/feature/Ranger/test_ranger.cpp +++ b/src/test/feature/Ranger/test_ranger.cpp @@ -42,7 +42,7 @@ TEST_F(TestHawqRanger, BasicTest) { string rootPath(util.getTestRootPath()); string initfile = hawq::test::stringFormat("Ranger/sql/init_file"); - auto cmd = hawq::test::stringFormat("ls -l %s/Ranger/sql/normal/*.sql 2>/dev/null | wc -l", rootPath.c_str()); + auto cmd = hawq::test::stringFormat("ls -l %s/Ranger/sql/normal/*.sql 2>/dev/null | grep \"^-\" | wc -l", rootPath.c_str()); int sql_num = std::atoi(Command::getCommandOutput(cmd).c_str()); int writableTableCase = 28; string rangerHost = RANGER_HOST; @@ -88,10 +88,10 @@ TEST_F(TestHawqRanger, BasicTest) { string super_ansfile_fail = hawq::test::stringFormat("Ranger/ans/super%d_fail.ans", i); string admin_ansfile = hawq::test::stringFormat("Ranger/ans/adminfirst%d.ans", i); - cmd = hawq::test::stringFormat("ls -l %s/Ranger/policy/%d/ 2>/dev/null| wc -l", rootPath.c_str(), i); + cmd = hawq::test::stringFormat("ls -l %s/Ranger/policy/%d/ 2>/dev/null| grep \"^-\" | wc -l", rootPath.c_str(), i); int policy_num = std::atoi(Command::getCommandOutput(cmd).c_str()); - cmd = hawq::test::stringFormat("ls -l %s/Ranger/sql/super/%d.sql 2>/dev/null| wc -l", rootPath.c_str(), i); + cmd = hawq::test::stringFormat("ls -l %s/Ranger/sql/super/%d.sql 2>/dev/null| grep \"^-\" | wc -l", rootPath.c_str(), i); int supersqlexist = std::atoi(Command::getCommandOutput(cmd).c_str()); if (policy_num > 0){ @@ -122,9 +122,9 @@ TEST_F(TestHawqRanger, BasicTest) { string super_ansfile_success = hawq::test::stringFormat("Ranger/ans/super%d_success.ans", i); - cmd = hawq::test::stringFormat("ls -l %s/Ranger/policy/%d/ 2>/dev/null| wc -l", rootPath.c_str(), i); + cmd = hawq::test::stringFormat("ls -l %s/Ranger/policy/%d/ 2>/dev/null| grep \"^-\" | wc -l", rootPath.c_str(), i); int policy_num = std::atoi(Command::getCommandOutput(cmd).c_str()); - cmd = hawq::test::stringFormat("ls -l %s/Ranger/sql/super/%d.sql 2>/dev/null | wc -l", rootPath.c_str(), i); + cmd = hawq::test::stringFormat("ls -l %s/Ranger/sql/super/%d.sql 2>/dev/null | grep \"^-\" | wc -l", rootPath.c_str(), i); int supersqlexist = std::atoi(Command::getCommandOutput(cmd).c_str()); util.execSQLFile(normal_sqlfile, normal_ansfile_success, initfile, true, true); if (supersqlexist) { @@ -146,7 +146,7 @@ TEST_F(TestHawqRanger, BasicTest) { util.execute(hawq::test::stringFormat("drop role %s;",normalusername.c_str()), false); util.execute(hawq::test::stringFormat("drop role %s;",superusername.c_str()), false); // delete policy - std::string cmd = hawq::test::stringFormat("ls -l %s/Ranger/policy/%d/ 2>/dev/null | wc -l", rootPath.c_str(), i); + std::string cmd = hawq::test::stringFormat("ls -l %s/Ranger/policy/%d/ 2>/dev/null | grep \"^-\" | wc -l", rootPath.c_str(), i); int policy_num = std::atoi(Command::getCommandOutput(cmd).c_str()); for (int j = 1; j <= policy_num; j++) { cmd = hawq::test::stringFormat("python %s/Ranger/rangerpolicy.py -h %s -d policy%d-%d", rootPath.c_str(), rangerHost.c_str(), i, j); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68013fac/src/test/feature/gtest-parallel ---------------------------------------------------------------------- diff --git a/src/test/feature/gtest-parallel b/src/test/feature/gtest-parallel index 7d8570a..19c0aed 100755 --- a/src/test/feature/gtest-parallel +++ b/src/test/feature/gtest-parallel @@ -469,12 +469,6 @@ def start_daemon(func): return t -#run test in parallel way -workers = [start_daemon(worker) for i in range(options.workers)] - -[t.join() for t in workers] - - #run test in serail way job_id = parallel_id while True: @@ -488,6 +482,13 @@ while True: break times.record_test_time(test_binary, test, run_job(job)) + +#run test in parallel way +workers = [start_daemon(worker) for i in range(options.workers)] + +[t.join() for t in workers] + + logger.end() times.write_to_file(save_file) if options.print_test_times:
