morningman commented on code in PR #68220: URL: https://github.com/apache/doris/pull/68220#discussion_r4069577239
########## fe/fe-authorization/fe-authorization-plugins/fe-authorization-plugin-ranger-common/src/main/java/org/apache/doris/catalog/authorizer/ranger/RangerUserStoreGroups.java: ########## @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog.authorizer.ranger; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig; +import org.apache.ranger.plugin.contextenricher.RangerAdminUserStoreRetriever; +import org.apache.ranger.plugin.contextenricher.RangerUserStoreEnricher; +import org.apache.ranger.plugin.policyengine.RangerPluginContext; +import org.apache.ranger.plugin.service.RangerAuthContext; +import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.apache.ranger.plugin.util.RangerUserStoreUtil; +import org.apache.ranger.plugin.util.ServiceDefUtil; +import org.apache.ranger.plugin.util.ServicePolicies; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** + * Where a request built by a Doris source gets the groups a Ranger policy item may be written against. + * + * <p>A Ranger policy item names users, groups and roles, and Ranger's own plugins fill in the groups from + * the service they run in - the Hive plugin asks Hadoop's group mapping. Doris has nothing to ask: an + * account holds roles, and no part of the engine knows which groups a user is in. So a request that carries + * no groups matches no group item at all, and a service whose policies are kept by group - the normal way + * to run Ranger, so that nobody edits a policy each time somebody joins a team - grants nothing to a Doris + * user; and, worse, denies nothing either. An item that denies a group is exactly as silent about it. + * + * <p>What Ranger provides for a plugin in that position is its <b>user store</b>: the users and groups Ranger + * Admin itself holds, kept current by usersync from LDAP or the OS, which a plugin downloads next to its + * policies through a {@link RangerUserStoreEnricher}. Ranger 2.5 and later reads the groups of the request's + * user out of it when {@code ranger.plugin.<type>.use.rangerGroups} is set; the two sources Doris embeds do + * the same thing here, in the request builder, so that it neither depends on the Ranger version they were + * built against - branches still on 2.4 have no such setting - nor on an operator finding a Ranger property + * no Doris document names. + * + * <p>It is on by default, because a group item that is silently ignored is a bug in the deployment's eyes, + * not a behaviour anybody opted into; the property Ranger reads for this is the one that switches it off, + * so that a deployment which has already decided the question in Ranger's terms has decided it here too. + * Switched off, requests carry no groups and this source matches what it matched before. + */ +public final class RangerUserStoreGroups { + private static final Logger LOG = LogManager.getLogger(RangerUserStoreGroups.class); + + /** + * Suffix of the property switching this off, {@code ranger.plugin.<type>.use.rangerGroups} in full: the + * name Ranger itself reads for the same thing, so that one setting decides both. + */ + public static final String USE_RANGER_GROUPS = ".use.rangerGroups"; + + /** How often the plugin asks Ranger Admin for a newer user store, when nothing configures it. */ + private static final String DEFAULT_REFRESH_INTERVAL_MS = Integer.toString(60 * 1000); + + private RangerUserStoreGroups() { + } + + /** Whether the plugin configured by {@code config} attaches user store groups; on unless switched off. */ + public static boolean enabledFor(RangerPluginConfig config) { + // No configuration at all - a plugin stubbed out in a test - has not switched anything off. + return config == null || config.getBoolean(config.getPropertyPrefix() + USE_RANGER_GROUPS, true); + } + + /** + * Puts a user store enricher on the service definition Ranger just downloaded, unless one is there or + * this is switched off, so that the plugin fetches and refreshes the user store {@link #groupsOf} reads. + * + * <p>Called by {@link LoadedRangerPlugin#setPolicies} before handing the policies on, on every + * call and not only the first: a download of policy deltas comes with its own copy of the service + * definition, which is why {@code RangerBasePlugin} re-adds the enricher on deltas too. The retriever + * class and the refresh interval are read under the option names Ranger itself reads them under, so that + * an operator who has tuned them for Ranger's own {@code use.rangerGroups} has tuned them here. + */ + public static void addUserStoreEnricher(RangerPluginConfig config, ServicePolicies policies) { + if (policies == null || config == null || !enabledFor(config)) { + return; + } + String retriever = config.get(RangerUserStoreEnricher.USERSTORE_RETRIEVER_CLASSNAME_OPTION, + RangerAdminUserStoreRetriever.class.getCanonicalName()); + String refreshIntervalMs = config.get(RangerUserStoreEnricher.USERSTORE_REFRESHER_POLLINGINTERVAL_OPTION, + DEFAULT_REFRESH_INTERVAL_MS); + // Ranger logs the addition itself, once per download that needed it; the operator-facing line about + // why the store is downloaded at all is the plugin's, written once when it starts (see describe). + if (ServiceDefUtil.addUserStoreEnricher(policies, retriever, refreshIntervalMs) && LOG.isDebugEnabled()) { + LOG.debug("Ranger service {} will download its user store every {} ms", policies.getServiceName(), + refreshIntervalMs); + } + } + + /** One line for the plugin's start-up log, saying what this does for it and how to switch it off. */ + public static String describe(RangerPluginConfig config) { + String property = config.getPropertyPrefix() + USE_RANGER_GROUPS; + return enabledFor(config) + ? "Ranger service " + config.getServiceName() + ": requests carry the groups Ranger's user store" + + " puts the user in, so that policy items written against a group apply; set " + + property + "=false to switch that off" + : "Ranger service " + config.getServiceName() + ": " + property + "=false, so requests carry no" + + " groups and policy items written against a group never apply"; + } + + /** + * The groups the user store {@code plugin} has downloaded puts {@code user} in. + * + * <p>Empty when this is switched off, when no store has arrived - Ranger Admin could not be reached for + * it and nothing was cached, which {@link LoadedRangerPlugin} says why it does not refuse - and when the + * store does not know the user, which is the case for every account that exists in Doris only. Empty + * and not null on purpose: a request with an empty group set matches items written against users and + * roles exactly as it did before. + */ + public static Set<String> groupsOf(RangerBasePlugin plugin, String user) { + if (user == null || !enabledFor(plugin.getConfig())) { + return Collections.emptySet(); + } + // The auth context the policy engine publishes is where Ranger's own request processing reads the + // store from; it is replaced together with the engine, and the store is carried over when it is. + RangerPluginContext pluginContext = plugin.getPluginContext(); + RangerAuthContext authContext = pluginContext == null ? null : pluginContext.getAuthContext(); + RangerUserStoreUtil userStore = authContext == null ? null : authContext.getUserStoreUtil(); + Set<String> groups = userStore == null ? null : userStore.getUserGroups(user); Review Comment: Not changing, in three parts. `convert.emailToUser` has nothing to convert on a Doris request: a Doris user name cannot contain `@` (`FeNameFormat.USER_NAME_REGEX`), so the request never carries an email the store would have to be asked about. `supports.name.transformation` is opt-in (default `false`) and nothing Doris documents. With it on, the store lookup by the untransformed name can miss a store keyed by the transformed one; the effect is that group items do not apply for that user, i.e. the behaviour before this PR, never a grant that Ranger's own path would not make. The NPE in `getTransformedGroups` (case conversion configured on the service, no group mapping handler) is Ranger's own bug: `getTransformedUser` guards `nameTransformer != null`, `getTransformedGroups` does not, on Ranger master too. In that configuration every request of Ranger's own plugins, whose requests always carry groups, throws the same way, which is why no deployment runs it. That is worth a RANGER JIRA; a Doris-side workaround for it is not. `use.rangerGroups=false` restores the previous requests for anyone who wants that in the meantime. ########## fe/fe-authorization/fe-authorization-plugins/fe-authorization-plugin-ranger-common/src/main/java/org/apache/doris/catalog/authorizer/ranger/RangerUserStoreGroups.java: ########## @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog.authorizer.ranger; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig; +import org.apache.ranger.plugin.contextenricher.RangerAdminUserStoreRetriever; +import org.apache.ranger.plugin.contextenricher.RangerUserStoreEnricher; +import org.apache.ranger.plugin.policyengine.RangerPluginContext; +import org.apache.ranger.plugin.service.RangerAuthContext; +import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.apache.ranger.plugin.util.RangerUserStoreUtil; +import org.apache.ranger.plugin.util.ServiceDefUtil; +import org.apache.ranger.plugin.util.ServicePolicies; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** + * Where a request built by a Doris source gets the groups a Ranger policy item may be written against. + * + * <p>A Ranger policy item names users, groups and roles, and Ranger's own plugins fill in the groups from + * the service they run in - the Hive plugin asks Hadoop's group mapping. Doris has nothing to ask: an + * account holds roles, and no part of the engine knows which groups a user is in. So a request that carries + * no groups matches no group item at all, and a service whose policies are kept by group - the normal way + * to run Ranger, so that nobody edits a policy each time somebody joins a team - grants nothing to a Doris + * user; and, worse, denies nothing either. An item that denies a group is exactly as silent about it. + * + * <p>What Ranger provides for a plugin in that position is its <b>user store</b>: the users and groups Ranger + * Admin itself holds, kept current by usersync from LDAP or the OS, which a plugin downloads next to its + * policies through a {@link RangerUserStoreEnricher}. Ranger 2.5 and later reads the groups of the request's + * user out of it when {@code ranger.plugin.<type>.use.rangerGroups} is set; the two sources Doris embeds do + * the same thing here, in the request builder, so that it neither depends on the Ranger version they were + * built against - branches still on 2.4 have no such setting - nor on an operator finding a Ranger property + * no Doris document names. + * + * <p>It is on by default, because a group item that is silently ignored is a bug in the deployment's eyes, + * not a behaviour anybody opted into; the property Ranger reads for this is the one that switches it off, + * so that a deployment which has already decided the question in Ranger's terms has decided it here too. + * Switched off, requests carry no groups and this source matches what it matched before. + */ +public final class RangerUserStoreGroups { + private static final Logger LOG = LogManager.getLogger(RangerUserStoreGroups.class); + + /** + * Suffix of the property switching this off, {@code ranger.plugin.<type>.use.rangerGroups} in full: the + * name Ranger itself reads for the same thing, so that one setting decides both. + */ + public static final String USE_RANGER_GROUPS = ".use.rangerGroups"; + + /** How often the plugin asks Ranger Admin for a newer user store, when nothing configures it. */ + private static final String DEFAULT_REFRESH_INTERVAL_MS = Integer.toString(60 * 1000); + + private RangerUserStoreGroups() { + } + + /** Whether the plugin configured by {@code config} attaches user store groups; on unless switched off. */ + public static boolean enabledFor(RangerPluginConfig config) { + // No configuration at all - a plugin stubbed out in a test - has not switched anything off. + return config == null || config.getBoolean(config.getPropertyPrefix() + USE_RANGER_GROUPS, true); Review Comment: Fixed in 7ca5ef5c49a. `RangerUserStoreGroups.enabledFor` now reads the property strictly - unset, `true` or `false`, case-insensitive, trimmed - and `validate()` refuses anything else before `RangerBasePlugin.init()` starts, naming the property and the value. Covered by `RangerUserStoreGroupsTest.testRefusesASwitchThatIsNeitherTrueNorFalse` and `LoadedRangerPluginTest.testRefusedBeforeTheLoadForASwitchThatIsNeitherTrueNorFalse`. ########## fe/fe-authorization/fe-authorization-plugins/fe-authorization-plugin-ranger-hive/src/main/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessController.java: ########## @@ -343,6 +343,9 @@ private RangerAccessRequestImpl createRequest(AuthorizedSubject subject, Set<Str AccessContext context) { RangerAccessRequestImpl request = new RangerAccessRequestImpl(); request.setUser(subject.getUser()); + // Ranger's groups for the user, out of the user store this service's plugin downloads; a Hive + // service's policies are usually kept by group, and Doris has no groups of its own to send. + request.setUserGroups(groupsOf(subject)); Review Comment: Not changing. Ranger's own `use.rangerGroups` path (`RangerDefaultRequestProcessor.updateUserGroups`) reads the store per request inside the very same `isAccessAllowed(Collection)` batch, and its role derivation (`getRolesForUserAndGroups`) is per request as well - which is what the ranger-doris source relies on today. "One membership generation per logical authorization" is not a property Ranger offers anywhere; the window here is one store swap inside a loop that takes microseconds, and the outcome is the union of two memberships each of which Ranger would have granted on its own. The roles snapshot in the Hive batch exists for the read lock the engine takes to answer them, not for consistency. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
