Repository: incubator-hawq Updated Branches: refs/heads/master a80e9e8ca -> 6c012e999
HAWQ-1380. Keep hawq_toolkit schema check in HAWQ native side Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6c012e99 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6c012e99 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6c012e99 Branch: refs/heads/master Commit: 6c012e9994fdc27b6d515ecd63fa1525880f4aa4 Parents: a80e9e8 Author: interma <[email protected]> Authored: Wed Mar 8 12:41:30 2017 +0800 Committer: Wen Lin <[email protected]> Committed: Fri Mar 10 15:44:22 2017 +0800 ---------------------------------------------------------------------- src/backend/catalog/aclchk.c | 21 +++++-- src/backend/utils/misc/guc.c | 8 ++- src/include/utils/guc.h | 4 +- .../feature/Ranger/ans/normal10000_success.ans | 10 ++++ src/test/feature/Ranger/policy/10000/1.json | 1 + src/test/feature/Ranger/sql/manual/10000.sql | 5 ++ src/test/feature/Ranger/test_ranger.cpp | 61 ++++++++++++++++++-- src/test/feature/sanity_tests.txt | 2 +- 8 files changed, 97 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c012e99/src/backend/catalog/aclchk.c ---------------------------------------------------------------------- diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index 16e00c1..e3d4d61 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -2685,8 +2685,12 @@ List *getActionName(AclMode mask) bool checkNamespaceFallback(Oid x) { - if (x == PG_CATALOG_NAMESPACE || x == information_schema_namespcace_oid - || x == PG_AOSEGMENT_NAMESPACE || x == PG_TOAST_NAMESPACE || x == PG_BITMAPINDEX_NAMESPACE) + if (x == PG_CATALOG_NAMESPACE + || x == PG_AOSEGMENT_NAMESPACE + || x == PG_TOAST_NAMESPACE + || x == PG_BITMAPINDEX_NAMESPACE + || x == information_schema_namespace_oid + || x == hawq_toolkit_schema_namespace_oid ) { return true; } @@ -2707,13 +2711,18 @@ bool checkNamespaceFallback(Oid x) bool fallBackToNativeCheck(AclObjectKind objkind, Oid obj_oid, Oid roleid, AclMode mode) { - /* get the latest information_schema_namespcace_oid. Since caql access heap table - * directly without aclcheck, this function will not be called recursively + /* get the latest information_schema_namespace_oid and hawq_toolkit_schema_namespace_oid. + * Since caql access heap table directly without aclcheck, this function will not be called recursively */ - if (information_schema_namespcace_oid == 0) + if (information_schema_namespace_oid == 0) { - information_schema_namespcace_oid = (int)get_namespace_oid("information_schema"); + information_schema_namespace_oid = (int)get_namespace_oid("information_schema"); } + if (hawq_toolkit_schema_namespace_oid == 0) + { + hawq_toolkit_schema_namespace_oid = (int)get_namespace_oid("hawq_toolkit"); + } + /* for heap table, we fall back to native check. */ if (objkind == ACL_KIND_CLASS) { http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c012e99/src/backend/utils/misc/guc.c ---------------------------------------------------------------------- diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index bd03d5e..2c2b918 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -768,7 +768,9 @@ bool optimizer_prefer_scalar_dqa_multistage_agg; bool optimizer_parallel_union; bool optimizer_array_constraints; -int information_schema_namespcace_oid; +/* fallback in ranger ACL check */ +int information_schema_namespace_oid; +int hawq_toolkit_schema_namespace_oid; /* Security */ bool gp_reject_internal_tcp_conn = true; @@ -6188,11 +6190,11 @@ static struct config_int ConfigureNamesInt[] = }, { - {"information_schema_namespcace_oid", PGC_USERSET, DEVELOPER_OPTIONS, + {"information_schema_namespace_oid", PGC_USERSET, DEVELOPER_OPTIONS, gettext_noop("the oid of information_schema namespace"), NULL }, - &information_schema_namespcace_oid, + &information_schema_namespace_oid, 0, 0, INT_MAX, NULL, NULL }, http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c012e99/src/include/utils/guc.h ---------------------------------------------------------------------- diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index 77cee1e..86154ee 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -445,7 +445,9 @@ extern bool optimizer_parallel_union; extern bool optimizer_array_constraints; -extern int information_schema_namespcace_oid; +/* fallback in ranger ACL check */ +extern int information_schema_namespace_oid; +extern int hawq_toolkit_schema_namespace_oid; /** * Enable logging of DPE match in optimizer. http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c012e99/src/test/feature/Ranger/ans/normal10000_success.ans ---------------------------------------------------------------------- diff --git a/src/test/feature/Ranger/ans/normal10000_success.ans b/src/test/feature/Ranger/ans/normal10000_success.ans new file mode 100644 index 0000000..125e649 --- /dev/null +++ b/src/test/feature/Ranger/ans/normal10000_success.ans @@ -0,0 +1,10 @@ +-- start_ignore +-- end_ignore +set session role=usertest10000; +SET +select count(*) from information_schema.view_table_usage; +0 +select count(*) from hawq_toolkit.hawq_table_indexes; +0 +select count(*) from pg_catalog.pg_compression; +4 http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c012e99/src/test/feature/Ranger/policy/10000/1.json ---------------------------------------------------------------------- diff --git a/src/test/feature/Ranger/policy/10000/1.json b/src/test/feature/Ranger/policy/10000/1.json new file mode 100644 index 0000000..e3a4d86 --- /dev/null +++ b/src/test/feature/Ranger/policy/10000/1.json @@ -0,0 +1 @@ +{"allowExceptions": [], "denyExceptions": [], "denyPolicyItems": [], "description": "no description", "isAuditEnabled": true, "isEnabled": true, "name": "policy10000-1", "policyItems": [{"accesses": [{"isAllowed": true, "type": "usage-schema"}, {"isAllowed": true, "type": "create"}], "conditions": [], "delegateAdmin": true, "groups": null, "users": ["usertest10000", "usersuper10000"]}], "resources": {"database": {"isExcludes": false, "isRecursive": false, "values": ["hawq_feature_test_db"]}, "schema": {"isExcludes": false, "isRecursive": false, "values": ["public"]}, "table": {"isExcludes": false, "isRecursive": false, "values": ["*"]}}, "service": "hawq", "version": 1} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c012e99/src/test/feature/Ranger/sql/manual/10000.sql ---------------------------------------------------------------------- diff --git a/src/test/feature/Ranger/sql/manual/10000.sql b/src/test/feature/Ranger/sql/manual/10000.sql new file mode 100644 index 0000000..3912cb3 --- /dev/null +++ b/src/test/feature/Ranger/sql/manual/10000.sql @@ -0,0 +1,5 @@ +set session role=usertest10000; +select count(*) from information_schema.view_table_usage; +select count(*) from hawq_toolkit.hawq_table_indexes; +select count(*) from pg_catalog.pg_compression; + http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c012e99/src/test/feature/Ranger/test_ranger.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/Ranger/test_ranger.cpp b/src/test/feature/Ranger/test_ranger.cpp index acc8b97..93e8452 100644 --- a/src/test/feature/Ranger/test_ranger.cpp +++ b/src/test/feature/Ranger/test_ranger.cpp @@ -41,7 +41,7 @@ TEST_F(TestHawqRanger, BasicTest) { gpdfist.init_gpfdist(); string rootPath(util.getTestRootPath()); - string initfile = hawq::test::stringFormat("Ranger/sql/init_file"); + string initfile = "Ranger/sql/init_file"; auto cmd = hawq::test::stringFormat("ls -l %s/Ranger/sql/normal/*.sql 2>/dev/null | grep \"^-\" | wc -l", rootPath.c_str()); int sql_num = std::atoi(Command::getCommandOutput(cmd).c_str()); int writableTableCase = 28; @@ -121,9 +121,6 @@ TEST_F(TestHawqRanger, BasicTest) { string normal_ansfile_success = hawq::test::stringFormat("Ranger/ans/normal%d_success.ans", i); string super_ansfile_success = hawq::test::stringFormat("Ranger/ans/super%d_success.ans", i); - - cmd = hawq::test::stringFormat("ls -l %s/Ranger/policy/%d/ 2>/dev/null| grep \"^-\" | wc -l", rootPath.c_str(), i); - int policy_num = std::atoi(Command::getCommandOutput(cmd).c_str()); cmd = hawq::test::stringFormat("ls -l %s/Ranger/sql/super/%d.sql 2>/dev/null | grep \"^-\" | wc -l", rootPath.c_str(), i); int supersqlexist = std::atoi(Command::getCommandOutput(cmd).c_str()); util.execSQLFile(normal_sqlfile, normal_ansfile_success, initfile, true, true); @@ -157,3 +154,59 @@ TEST_F(TestHawqRanger, BasicTest) { gpdfist.finalize_gpfdist(); } } + +static void clear_env(SQLUtility &util, int sql_id, string rootPath, string rangerHost) +{ + int i = sql_id; + // delete user_num + std::string normalusername = hawq::test::stringFormat("usertest%d", i); + std::string superusername = hawq::test::stringFormat("usersuper%d", i); + util.execute(hawq::test::stringFormat("drop role %s;",normalusername.c_str()), false); + util.execute(hawq::test::stringFormat("drop role %s;",superusername.c_str()), false); + + // delete policy + std::string cmd = hawq::test::stringFormat("ls -l %s/Ranger/policy/%d/ 2>/dev/null| grep \"^-\" | wc -l ", rootPath.c_str(), i); + int policy_num = std::atoi(Command::getCommandOutput(cmd).c_str()); + for (int j = 1; j <= policy_num; j++) { + cmd = hawq::test::stringFormat("python %s/Ranger/rangerpolicy.py -h %s -d policy%d-%d", rootPath.c_str(), rangerHost.c_str(), i, j); + Command::getCommandStatus(cmd); + } +} + +TEST_F(TestHawqRanger, FallbackTest) { + SQLUtility util; + + if (util.getGUCValue("hawq_acl_type") == "ranger") + { + string rootPath(util.getTestRootPath()); + string rangerHost = RANGER_HOST; + string initfile = "Ranger/sql/init_file"; + string cmd; + int FallbackCase = 10000; + int i = FallbackCase; + + // clear environment + clear_env(util, i, rootPath, rangerHost); + + // create user_num + std::string normalusername = hawq::test::stringFormat("usertest%d", i);; + std::string superusername = hawq::test::stringFormat("usersuper%d", i);; + util.execute(hawq::test::stringFormat("create role %s with login createdb;", normalusername.c_str()),true); + util.execute(hawq::test::stringFormat("create role %s with login createdb superuser;", superusername.c_str()),true); + // add user + cmd = hawq::test::stringFormat("python %s/Ranger/rangeruser.py -h %s -u %s,%s", rootPath.c_str(), + rangerHost.c_str(), normalusername.c_str(), superusername.c_str()); + Command::getCommandStatus(cmd); + // add policy + cmd = hawq::test::stringFormat("python %s/Ranger/rangerpolicy.py -h %s -a %s/Ranger/policy/%d/%d.json", rootPath.c_str(), rangerHost.c_str(), rootPath.c_str(), i, 1); + Command::getCommandStatus(cmd); + sleep(60); + + // run sql test + string normal_sqlfile = hawq::test::stringFormat("Ranger/sql/manual/%d.sql", i); + string normal_ansfile_success = hawq::test::stringFormat("Ranger/ans/normal%d_success.ans", i); + util.execSQLFile(normal_sqlfile, normal_ansfile_success, initfile, true, true); + + clear_env(util, i, rootPath, rangerHost); + } +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c012e99/src/test/feature/sanity_tests.txt ---------------------------------------------------------------------- diff --git a/src/test/feature/sanity_tests.txt b/src/test/feature/sanity_tests.txt index 1c58281..bc6df0b 100644 --- a/src/test/feature/sanity_tests.txt +++ b/src/test/feature/sanity_tests.txt @@ -3,4 +3,4 @@ #you can have several PARALLEL or SRRIAL PARALLEL=TestErrorTable.*:TestPreparedStatement.*:TestUDF.*:TestAOSnappy.*:TestAlterOwner.*:TestAlterTable.*:TestCreateTable.*:TestGuc.*:TestType.*:TestDatabase.*:TestParquet.*:TestPartition.*:TestSubplan.*:TestAggregate.*:TestCreateTypeComposite.*:TestGpDistRandom.*:TestInformationSchema.*:TestQueryInsert.*:TestQueryNestedCaseNull.*:TestQueryPolymorphism.*:TestQueryPortal.*:TestQueryPrepare.*:TestQuerySequence.*:TestCommonLib.*:TestToast.*:TestTransaction.*:TestCommand.*:TestCopy.*:TestHawqRegister.TestPartitionTableMultilevel:TestHawqRegister.TestUsage1ExpectSuccessDifferentSchema:TestHawqRegister.TestUsage1ExpectSuccess:TestHawqRegister.TestUsage1SingleHawqFile:TestHawqRegister.TestUsage1SingleHiveFile:TestHawqRegister.TestDataTypes:TestHawqRegister.TestUsage1EofSuccess:TestHawqRegister.TestUsage2Case1Expected:TestHawqRegister.TestUsage2Case2Expected -SERIAL=TestHawqRanger.BasicTest:TestExternalOid.TestExternalOidAll:TestExternalTable.TestExternalTableAll:TestTemp.BasicTest:TestRowTypes.* +SERIAL=TestHawqRanger.*:TestExternalOid.TestExternalOidAll:TestExternalTable.TestExternalTableAll:TestTemp.BasicTest:TestRowTypes.*
