This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch ranger-2.6
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/ranger-2.6 by this push:
new eac0d3f77 RANGER-4889: updated RangerMemSizing to support access
request evaluations
eac0d3f77 is described below
commit eac0d3f779e6788772934f1c667df3a98369e67e
Author: Madhan Neethiraj <[email protected]>
AuthorDate: Mon Aug 5 18:08:53 2024 -0700
RANGER-4889: updated RangerMemSizing to support access request evaluations
(cherry picked from commit 9de9676d5d2100bcba2574e507c9d4ba474ba0f3)
---
.../apache/ranger/sizing/PerfRequestGenerator.java | 160 +++++++++++
.../org/apache/ranger/sizing/RangerMemSizing.java | 307 ++++++++++++++++++++-
2 files changed, 464 insertions(+), 3 deletions(-)
diff --git
a/ranger-tools/src/main/java/org/apache/ranger/sizing/PerfRequestGenerator.java
b/ranger-tools/src/main/java/org/apache/ranger/sizing/PerfRequestGenerator.java
new file mode 100644
index 000000000..a33299312
--- /dev/null
+++
b/ranger-tools/src/main/java/org/apache/ranger/sizing/PerfRequestGenerator.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.sizing;
+
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
+import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
+import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl;
+import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;
+import org.apache.ranger.plugin.util.ServicePolicies;
+import org.apache.ranger.plugin.util.ServiceTags;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
+
+public class PerfRequestGenerator {
+ public PerfRequestGenerator() {
+ }
+
+ public Collection<RangerAccessRequest> generate(ServicePolicies policies,
ServiceTags tags) {
+ List<RangerAccessRequest> ret = new ArrayList<>();
+ Set<Map<String, Object>> resources = new HashSet<>();
+ Set<String> accessTypes = new HashSet<>();
+ Set<String> users = new HashSet<>();
+ Set<String> groups = new HashSet<>();
+
+ // collect resources, users, groups referenced in policies
+ // collect accessTypes from serviceDef
+ if (policies != null) {
+ if (policies.getServiceDef() != null &&
policies.getServiceDef().getAccessTypes() != null) {
+ policies.getServiceDef().getAccessTypes().forEach(atd ->
accessTypes.add(atd.getName()));
+ }
+
+ if (policies.getPolicies() != null) {
+ policies.getPolicies().forEach(policy -> {
+ collectResources(policy.getResources(), resources);
+
+ if (policy.getAdditionalResources() != null) {
+ policy.getAdditionalResources().forEach(resource ->
collectResources(resource, resources));
+ }
+
+ collectUsersGroups(policy, users, groups);
+ });
+ }
+
+ if (policies.getTagPolicies() != null &&
policies.getTagPolicies().getPolicies() != null) {
+ policies.getTagPolicies().getPolicies().forEach(policy ->
collectUsersGroups(policy, users, groups));
+ }
+ }
+
+ // collect resources from tagged-resources
+ if (tags != null && tags.getServiceResources() != null) {
+ tags.getServiceResources().forEach(resource ->
collectResources(resource.getResourceElements(), resources));
+ }
+
+ if (accessTypes.isEmpty()) {
+ accessTypes.add("read");
+ }
+
+ if (users.isEmpty()) {
+ users.add("user1");
+ }
+
+ if (groups.isEmpty()) {
+ groups.add("group1");
+ }
+
+ Iterator<String> iterAccessTypes = accessTypes.iterator();
+ Iterator<String> iterUser = users.iterator();
+ Iterator<String> iterGroup = groups.iterator();
+
+ for (Map<String, Object> resource : resources) {
+ ret.add(new RangerAccessRequestImpl(new
RangerAccessResourceImpl(resource), iterAccessTypes.next(), iterUser.next(),
Collections.singleton(iterGroup.next()), null));
+
+ if (!iterAccessTypes.hasNext()) {
+ iterAccessTypes = accessTypes.iterator();
+ }
+
+ if (!iterUser.hasNext()) {
+ iterUser = users.iterator();
+ }
+
+ if (!iterGroup.hasNext()) {
+ iterGroup = groups.iterator();
+ }
+ }
+
+ return ret;
+ }
+
+ private void collectResources(Map<String, RangerPolicyResource>
policyResource, Set<Map<String, Object>> resources) {
+ if (policyResource != null) {
+ int resourceCount = 1;
+
+ for (Map.Entry<String, RangerPolicyResource> entry :
policyResource.entrySet()) {
+ resourceCount *= entry.getValue().getValues().size();
+ }
+
+ List<Map<String, Object>> toAdd = new ArrayList<>(resourceCount);
+
+ for (int i = 0; i < resourceCount; i++) {
+ toAdd.add(new HashMap<>());
+ }
+
+ for (Map.Entry<String, RangerPolicyResource> entry :
policyResource.entrySet()) {
+ String name = entry.getKey();
+ List<String> values = entry.getValue().getValues();
+
+ for (int idxResource = 0; idxResource < resourceCount;
idxResource++) {
+ toAdd.get(idxResource).put(name, values.get(idxResource %
values.size()));
+ }
+ }
+
+ resources.addAll(toAdd);
+ }
+ }
+
+ private void collectUsersGroups(RangerPolicy policy, Set<String> users,
Set<String> groups) {
+ collectUsersGroups(policy.getPolicyItems(), users, groups);
+ collectUsersGroups(policy.getAllowExceptions(), users, groups);
+ collectUsersGroups(policy.getDenyPolicyItems(), users, groups);
+ collectUsersGroups(policy.getDenyExceptions(), users, groups);
+ collectUsersGroups(policy.getDataMaskPolicyItems(), users, groups);
+ collectUsersGroups(policy.getRowFilterPolicyItems(), users, groups);
+ }
+
+ private void collectUsersGroups(List<? extends RangerPolicyItem>
policyItems, Set<String> users, Set<String> groups) {
+ if (policyItems != null) {
+ policyItems.forEach(policyItem -> {
+ users.addAll(policyItem.getUsers());
+ groups.addAll(policyItem.getGroups());
+ });
+ }
+ }
+}
diff --git
a/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java
b/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java
index 74f231013..ee9d8b47f 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java
@@ -20,14 +20,42 @@
package org.apache.ranger.sizing;
import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.text.SimpleDateFormat;
+import java.util.Collection;
+import java.util.Collections;
import java.util.Date;
+import java.util.List;
+import java.util.Map;
import java.util.Set;
-
-import org.apache.commons.cli.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.Version;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
import org.apache.commons.lang.StringUtils;
import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig;
import org.apache.ranger.authorization.utils.JsonUtils;
+import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
+import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl;
+import org.apache.ranger.plugin.policyengine.RangerAccessResource;
+import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;
+import org.apache.ranger.plugin.policyengine.RangerAccessResult;
import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions;
import org.apache.ranger.plugin.service.RangerBasePlugin;
import org.apache.ranger.plugin.util.RangerRoles;
@@ -43,10 +71,15 @@ public class RangerMemSizing {
private final SimpleDateFormat DATE_FORMAT = new
SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S");
+ private static final TypeReference<List<RangerAccessRequest>>
TYPE_LIST_REQUESTS = new TypeReference<List<RangerAccessRequest>>() {};
+
private final String policyFile;
private final String tagFile;
private final String rolesFile;
private final String userStoreFile;
+ private final String genRequestsFile;
+ private final String evalRequestsFile;
+ private final int evalClientsCount;
private final boolean deDup;
private final boolean deDupStrings;
private final String optimizationMode;
@@ -58,6 +91,9 @@ public class RangerMemSizing {
this.tagFile = cmdLine.getOptionValue('t');
this.rolesFile = cmdLine.getOptionValue('r');
this.userStoreFile = cmdLine.getOptionValue('u');
+ this.genRequestsFile = cmdLine.getOptionValue('q');
+ this.evalRequestsFile = cmdLine.getOptionValue('e');
+ this.evalClientsCount = cmdLine.hasOption('c') ?
Integer.parseInt(cmdLine.getOptionValue('c')) : 1;
this.deDup = Boolean.parseBoolean(cmdLine.getOptionValue("d",
"true"));
this.deDupStrings = this.deDup;
this.optimizationMode =
StringUtils.startsWithIgnoreCase(cmdLine.getOptionValue('o', "space"), "s") ?
OPT_MODE_SPACE : OPT_MODEL_RETRIEVAL;
@@ -71,6 +107,20 @@ public class RangerMemSizing {
RangerRoles roles = loadRoles(rolesFile, tracker);
RangerUserStore userStore = loadUserStore(userStoreFile, tracker);
RangerBasePlugin plugin = createRangerPlugin(policies, tags, roles,
userStore, tracker);
+ int genReqCount = 0;
+ int evalReqCount = 0;
+ int evalAvgTimeNs = 0;
+
+ if (StringUtils.isNotBlank(genRequestsFile)) {
+ genReqCount = generateRequestsFile(policies, tags, genRequestsFile,
tracker);
+ }
+
+ if (StringUtils.isNotBlank(evalRequestsFile)) {
+ int[] res = evaluateRequests(evalRequestsFile, plugin, tracker);
+
+ evalReqCount = res[0];
+ evalAvgTimeNs = res[1];
+ }
tracker.stop();
@@ -92,6 +142,14 @@ public class RangerMemSizing {
out.println(" UserStore: file=" + userStoreFile + ", size=" + new
File(userStoreFile).length() + ", " + toSummaryStr(userStore));
}
+ if (genRequestsFile != null) {
+ out.println(" GenReq: file=" + genRequestsFile + ", requestCount=" +
genReqCount);
+ }
+
+ if (evalRequestsFile != null) {
+ out.println(" EvalReq: file=" + evalRequestsFile + ", requestCount="
+ evalReqCount + ", avgTimeTaken=" + evalAvgTimeNs + "ns, clientCount=" +
evalClientsCount);
+ }
+
out.println(" DeDup: " + deDup);
out.println(" OptMode: " + optimizationMode);
out.println();
@@ -314,13 +372,144 @@ public class RangerMemSizing {
return ret;
}
+ private int generateRequestsFile(ServicePolicies policies, ServiceTags tags,
String fileName, PerfMemTimeTracker parent) {
+ ObjectMapper mapper = JsonUtils.getMapper();
+
+ initMapper(mapper);
+
+ PerfMemTimeTracker tracker = new PerfMemTimeTracker("generateRequests");
+
+ Collection<RangerAccessRequest> requests = new
PerfRequestGenerator().generate(policies, tags);
+
+ log("generateRequestsFile(): saving " + requests.size() + " requests..");
+
+ try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(fileName)))
{
+ writer.write('[');
+
+ boolean isFirst = true;
+
+ for (RangerAccessRequest request : requests) {
+ if (!isFirst) {
+ writer.write(',');
+ } else {
+ isFirst = false;
+ }
+ writer.newLine();
+
+ writer.write(mapper.writeValueAsString(request));
+ }
+
+ writer.newLine();
+ writer.write(']');
+
+ log("generateRequestsFile(): saved " + requests.size() + " requests");
+ } catch (IOException excp) {
+ log("generateRequestsFile(): failed", excp);
+ }
+
+ tracker.stop();
+ parent.addChild(tracker);
+
+ return requests.size();
+ }
+
+ private int[] evaluateRequests(String evalRequestsFile, RangerBasePlugin
plugin, PerfMemTimeTracker parent) {
+ PerfMemTimeTracker tracker = new PerfMemTimeTracker("evaluateRequests");
+
+ final List<RangerAccessRequest> requests =
readRequests(evalRequestsFile, tracker);
+ final Thread[] clients = new
Thread[evalClientsCount];
+ final AtomicInteger idxNextRequest = new AtomicInteger();
+ final AtomicLong totalTimeTakenNs = new AtomicLong();
+
+ for (int idxClient = 0; idxClient < evalClientsCount; idxClient++) {
+ clients[idxClient] = new Thread(() -> {
+ do {
+ int idxReq = idxNextRequest.getAndIncrement();
+
+ if (idxReq >= requests.size()) {
+ break;
+ }
+
+ RangerAccessRequest request = requests.get(idxReq);
+
+ long startTime = System.nanoTime();
+
+ RangerAccessResult result = plugin.isAccessAllowed(request);
+
+ totalTimeTakenNs.getAndAdd(System.nanoTime() - startTime);
+
+ if ((idxReq + 1) % 1000 == 0) {
+ log(" evaluated requests: " + (idxReq + 1) + ", avgTimeTaken=" +
(totalTimeTakenNs.get() / idxReq) + "ns");
+ }
+
+ requests.set(idxReq, null); // so that objects associated with the
request can be freed
+ } while (true);
+ });
+ }
+
+ log("evaluateRequests(): evaluating " + requests.size() + " requests in "
+ clients.length + " clients...");
+
+ for (Thread client : clients) {
+ client.run();
+ }
+
+ log("evaluateRequests(): waiting for " + clients.length + " clients to
complete...");
+ for (Thread client : clients) {
+ try {
+ client.join();
+ } catch (Exception excp) {
+ // ignore
+ }
+ }
+
+ tracker.stop();
+ parent.addChild(tracker);
+
+ int avgTimeTakenNs = (int) (totalTimeTakenNs.get() / idxNextRequest.get());
+
+ log("evaluateRequests(): evaluated " + idxNextRequest.get() + " requests,
avgTimeTaken=" + avgTimeTakenNs + "ns");
+
+ return new int[] { requests.size(), avgTimeTakenNs };
+ }
+
+ private List<RangerAccessRequest> readRequests(String fileName,
PerfMemTimeTracker parent) {
+ PerfMemTimeTracker tracker = new PerfMemTimeTracker("readRequests");
+
+ List<RangerAccessRequest> ret = null;
+
+ ObjectMapper mapper = JsonUtils.getMapper();
+
+ initMapper(mapper);
+
+ try (InputStream inStr =
Files.newInputStream(Paths.get(evalRequestsFile))) {
+ ret = mapper.readValue(inStr, TYPE_LIST_REQUESTS);
+ } catch (IOException excp) {
+ log("readRequests(): failed to read file " + evalRequestsFile, excp);
+ }
+
+ if (ret == null) {
+ ret = Collections.emptyList();
+ }
+
+ tracker.stop();
+ parent.addChild(tracker);
+
+ log("readRequests(file=" + fileName + ", size=" + new
File(evalRequestsFile).length() + "): request count=" + ret.size());
+
+ return ret;
+ }
+
private static CommandLine parseArgs(String[] args) {
Option help = new Option("h", "help", false, "show help");
- Option deDup = new Option("d", "deDup", true, "deDup string/tags");
+ Option deDup = new Option("d", "deDup", true, "deDup string, tags:
true|false");
Option policies = new Option("p", "policies", true, "policies file");
Option tags = new Option("t", "tags", true, "tags file");
Option roles = new Option("r", "roles", true, "roles file");
Option userStore = new Option("u", "userStore", true, "userStore file");
+ Option genRequests = new Option("q", "genRequests", true, "generate
requests file");
+ Option evalRequests = new Option("e", "evalRequests", true, "eval requests
file");
+ Option evalClients = new Option("c", "evalClients", true, "eval clients
count");
+ Option gdsInfo = new Option("g", "gdsInfo", true, "gdsInfo file");
Option optimizeMode = new Option("o", "optMode", true, "optimization mode:
space|retrieval");
Options options = new Options();
@@ -330,6 +519,10 @@ public class RangerMemSizing {
options.addOption(tags);
options.addOption(roles);
options.addOption(userStore);
+ options.addOption(genRequests);
+ options.addOption(evalRequests);
+ options.addOption(evalClients);
+ options.addOption(gdsInfo);
options.addOption(deDup);
options.addOption(optimizeMode);
@@ -454,4 +647,112 @@ public class RangerMemSizing {
return "users=" + userCount + ", groups=" + groupCount + ",
userGroupMappings=" + userGroupCount;
}
+
+ private void initMapper(ObjectMapper mapper) {
+ SimpleModule serDeModule = new SimpleModule("RangerMemSizing", new
Version(1, 0, 0, null, null, null));
+
+ serDeModule.addSerializer(RangerAccessResource.class, new
RangerAccessResourceSerializer());
+ serDeModule.addSerializer(RangerAccessResourceImpl.class, new
RangerAccessResourceImplSerializer());
+ serDeModule.addDeserializer(RangerAccessResource.class, new
RangerAccessResourceDeserializer());
+
+ serDeModule.addSerializer(RangerAccessRequest.class, new
RangerAccessRequestSerializer());
+ serDeModule.addSerializer(RangerAccessRequestImpl.class, new
RangerAccessRequestImplSerializer());
+ serDeModule.addDeserializer(RangerAccessRequest.class, new
RangerAccessRequestDeserializer());
+
+ mapper.registerModule(serDeModule);
+ }
+
+
+ static class TestRangerAccessResourceImpl {
+ private Map<String, Object> elements;
+
+ public TestRangerAccessResourceImpl() {
+ }
+
+ public TestRangerAccessResourceImpl(Map<String, Object> elements) {
+ this.elements = elements;
+ }
+
+ public Map<String, Object> getElements() { return elements; }
+ }
+
+ static class TestRangerAccessRequestImpl {
+ private TestRangerAccessResourceImpl resource;
+ private String accessType;
+ private String user;
+ private Set<String> userGroups;
+
+ public TestRangerAccessRequestImpl() {
+ }
+
+ public TestRangerAccessRequestImpl(RangerAccessResource resource, String
accessType, String user, Set<String> userGroups) {
+ this.resource = new TestRangerAccessResourceImpl(resource.getAsMap());
+ this.accessType = accessType;
+ this.user = user;
+ this.userGroups = userGroups;
+ }
+
+ public TestRangerAccessResourceImpl getResource() {
+ return resource;
+ }
+
+ public String getAccessType() {
+ return accessType;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public Set<String> getUserGroups() {
+ return userGroups;
+ }
+
+ }
+
+ static class RangerAccessResourceSerializer extends
JsonSerializer<RangerAccessResource> {
+ @Override
+ public void serialize(RangerAccessResource value, JsonGenerator gen,
SerializerProvider serializers) throws IOException {
+ JsonUtils.getMapper().writeValue(gen, new
TestRangerAccessResourceImpl(value.getAsMap()));
+ }
+ }
+
+ static class RangerAccessResourceImplSerializer extends
JsonSerializer<RangerAccessResource> {
+ @Override
+ public void serialize(RangerAccessResource value, JsonGenerator gen,
SerializerProvider serializers) throws IOException {
+ JsonUtils.getMapper().writeValue(gen, new
TestRangerAccessResourceImpl(value.getAsMap()));
+ }
+ }
+
+ static class RangerAccessResourceDeserializer extends
JsonDeserializer<RangerAccessResource> {
+ @Override
+ public RangerAccessResource deserialize(JsonParser parser,
DeserializationContext context) throws IOException {
+ TestRangerAccessResourceImpl resource = context.readValue(parser,
TestRangerAccessResourceImpl.class);
+
+ return new RangerAccessResourceImpl(resource.getElements());
+ }
+ }
+
+ static class RangerAccessRequestSerializer extends
JsonSerializer<RangerAccessRequest> {
+ @Override
+ public void serialize(RangerAccessRequest value, JsonGenerator gen,
SerializerProvider serializers) throws IOException {
+ JsonUtils.getMapper().writeValue(gen, new
TestRangerAccessRequestImpl(value.getResource(), value.getAccessType(),
value.getUser(), value.getUserGroups()));
+ }
+ }
+
+ static class RangerAccessRequestImplSerializer extends
JsonSerializer<RangerAccessRequestImpl> {
+ @Override
+ public void serialize(RangerAccessRequestImpl value, JsonGenerator gen,
SerializerProvider serializers) throws IOException {
+ JsonUtils.getMapper().writeValue(gen, new
TestRangerAccessRequestImpl(value.getResource(), value.getAccessType(),
value.getUser(), value.getUserGroups()));
+ }
+ }
+
+ static class RangerAccessRequestDeserializer extends
JsonDeserializer<RangerAccessRequest> {
+ @Override
+ public RangerAccessRequest deserialize(JsonParser parser,
DeserializationContext context) throws IOException {
+ TestRangerAccessRequestImpl req = context.readValue(parser,
TestRangerAccessRequestImpl.class);
+
+ return new RangerAccessRequestImpl(new
RangerAccessResourceImpl(req.resource.getElements()), req.accessType, req.user,
req.userGroups, null);
+ }
+ }
}
