This is an automated email from the ASF dual-hosted git repository. michaelsmith pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 8f7d2246ec356334c28cb616c4c335d3a8d686fa Author: Fang-Yu Rao <[email protected]> AuthorDate: Tue Apr 29 16:07:50 2025 -0700 IMPALA-12554: (Addendum) Add a flag to not consolidate requests by default This patch adds a startup flag so that by default the catalog server will not consolidate the grant/revoke requests sent to the Ranger server when there are multiple columns involved in the GRANT/REVOKE statement. Testing: - Added 2 end-to-end tests to make sure the grant/revoke requests sent to the Ranger server would be consolidated only when the flag is explicitly added when we start the catalog server. Change-Id: I4defc59c048be1112380c3a7254ffa8655eee0af Reviewed-on: http://gerrit.cloudera.org:8080/22833 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- be/src/catalog/catalog.cc | 3 +++ be/src/util/backend-gflag-util.cc | 2 ++ common/thrift/BackendGflags.thrift | 2 ++ .../ranger/RangerCatalogdAuthorizationManager.java | 4 +++- .../main/java/org/apache/impala/service/BackendConfig.java | 4 ++++ tests/authorization/test_ranger.py | 13 ++++++++++++- 6 files changed, 26 insertions(+), 2 deletions(-) diff --git a/be/src/catalog/catalog.cc b/be/src/catalog/catalog.cc index cf329bcac..752aa6833 100644 --- a/be/src/catalog/catalog.cc +++ b/be/src/catalog/catalog.cc @@ -48,6 +48,9 @@ DEFINE_int32(initial_hms_cnxn_timeout_s, 120, "before exiting."); DEFINE_bool(enable_reading_puffin_stats, false, "Impala will only read Iceberg Puffin " "stats files if this flag is set to true."); +DEFINE_bool(consolidate_grant_revoke_requests, false, "Impala will create only one " + "Ranger policy when there are multiple columns in the GRANT statement if this flag " + "is set to true."); Catalog::Catalog() { JniMethodDescriptor methods[] = { diff --git a/be/src/util/backend-gflag-util.cc b/be/src/util/backend-gflag-util.cc index 462b3f1ce..c37b6afc5 100644 --- a/be/src/util/backend-gflag-util.cc +++ b/be/src/util/backend-gflag-util.cc @@ -142,6 +142,7 @@ DECLARE_int32(num_db_event_executors); DECLARE_int32(num_table_event_executors_per_db_event_executor); DECLARE_int32(min_event_processor_idle_ms); DECLARE_int32(max_outstanding_events_on_executors); +DECLARE_bool(consolidate_grant_revoke_requests); // HS2 SAML2.0 configuration // Defined here because TAG_FLAG caused issues in global-flags.cc @@ -532,6 +533,7 @@ Status PopulateThriftBackendGflags(TBackendGflags& cfg) { cfg.__set_min_event_processor_idle_ms(FLAGS_min_event_processor_idle_ms); cfg.__set_max_outstanding_events_on_executors( FLAGS_max_outstanding_events_on_executors); + cfg.__set_consolidate_grant_revoke_requests(FLAGS_consolidate_grant_revoke_requests); return Status::OK(); } diff --git a/common/thrift/BackendGflags.thrift b/common/thrift/BackendGflags.thrift index 5b88457e5..f1f93877c 100644 --- a/common/thrift/BackendGflags.thrift +++ b/common/thrift/BackendGflags.thrift @@ -337,4 +337,6 @@ struct TBackendGflags { 152: required i32 min_event_processor_idle_ms 153: required i32 max_outstanding_events_on_executors + + 154: required bool consolidate_grant_revoke_requests } diff --git a/fe/src/main/java/org/apache/impala/authorization/ranger/RangerCatalogdAuthorizationManager.java b/fe/src/main/java/org/apache/impala/authorization/ranger/RangerCatalogdAuthorizationManager.java index 3c9a02d59..344aab08d 100644 --- a/fe/src/main/java/org/apache/impala/authorization/ranger/RangerCatalogdAuthorizationManager.java +++ b/fe/src/main/java/org/apache/impala/authorization/ranger/RangerCatalogdAuthorizationManager.java @@ -30,6 +30,7 @@ import org.apache.impala.common.ImpalaException; import org.apache.impala.common.InternalException; import org.apache.impala.common.Pair; import org.apache.impala.common.UnsupportedFeatureException; +import org.apache.impala.service.BackendConfig; import org.apache.impala.thrift.TCatalogServiceRequestHeader; import org.apache.impala.thrift.TCreateDropRoleParams; import org.apache.impala.thrift.TDdlExecResponse; @@ -430,7 +431,8 @@ public class RangerCatalogdAuthorizationManager implements AuthorizationManager } } - return consolidateGrantRevokeRequests(requests); + return BackendConfig.INSTANCE.consolidateGrantRevokeRequests() ? + consolidateGrantRevokeRequests(requests) : requests; } private static GrantRevokeRequest createGrantRevokeRequest(String grantor, String user, diff --git a/fe/src/main/java/org/apache/impala/service/BackendConfig.java b/fe/src/main/java/org/apache/impala/service/BackendConfig.java index 8cbfcf4f7..d9d84719e 100644 --- a/fe/src/main/java/org/apache/impala/service/BackendConfig.java +++ b/fe/src/main/java/org/apache/impala/service/BackendConfig.java @@ -573,4 +573,8 @@ public class BackendConfig { public int getMaxOutstandingEventsOnExecutors() { return backendCfg_.max_outstanding_events_on_executors; } + + public boolean consolidateGrantRevokeRequests() { + return backendCfg_.consolidate_grant_revoke_requests; + } } diff --git a/tests/authorization/test_ranger.py b/tests/authorization/test_ranger.py index 20a62f8d4..b43bc4d9a 100644 --- a/tests/authorization/test_ranger.py +++ b/tests/authorization/test_ranger.py @@ -1263,6 +1263,17 @@ class TestRanger(CustomClusterTestSuite): @CustomClusterTestSuite.with_args( impalad_args=IMPALAD_ARGS, catalogd_args=CATALOGD_ARGS, reset_ranger=True) def test_grant_multiple_columns(self): + self._test_grant_multiple_columns(13) + + @pytest.mark.execute_serially + @CustomClusterTestSuite.with_args( + impalad_args=IMPALAD_ARGS, + catalogd_args="{0} {1}".format(CATALOGD_ARGS, "--consolidate_grant_revoke_requests"), + reset_ranger=True) + def test_grant_multiple_columns_consolidate_grant_revoke_requests(self): + self._test_grant_multiple_columns(1) + + def _test_grant_multiple_columns(self, expected_num_policies): admin_client = self.create_impala_client(user=ADMIN) access_type = "select" db = "functional" @@ -1297,7 +1308,7 @@ class TestRanger(CustomClusterTestSuite): # After the GRANT statement above, there should be only one single Ranger policy # that grants the privilege of 'access_type' on the column 'db'.'tbl'.'col' to # the principal 'principal_name' for each column in 'cols'. - assert len(policy_ids) == 1 + assert len(policy_ids) == expected_num_policies finally: admin_client.execute("revoke {0}({1}) on table {2}.{3} from {4} {5}" .format(access_type, cols_str, db, tbl, kw, principal_name))
