Repository: incubator-ranger Updated Branches: refs/heads/master 8b4ba8efc -> fec84603f
RANGER-796: good coding practice fixes in updates from RANGER-794 Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/fec84603 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/fec84603 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/fec84603 Branch: refs/heads/master Commit: fec84603f5139daea8730a4281400c2c426a234e Parents: 8b4ba8e Author: Abhay Kulkarni <[email protected]> Authored: Mon Jan 4 14:59:41 2016 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Wed Jan 6 18:06:09 2016 -0800 ---------------------------------------------------------------------- .../RangerFileBasedTagRetriever.java | 27 +++-- .../plugin/store/AbstractPredicateUtil.java | 25 ++--- .../ranger/plugin/util/PerfDataRecorder.java | 2 +- .../ranger/policyengine/CommandLineParser.java | 11 +- .../ranger/policyengine/PerfTestClient.java | 32 ++++-- .../ranger/policyengine/PerfTestEngine.java | 16 +-- .../RangerPolicyenginePerfTester.java | 105 ++++++++++--------- 7 files changed, 125 insertions(+), 93 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fec84603/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java index 95291be..db4c6e4 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java @@ -29,6 +29,7 @@ import org.apache.ranger.plugin.util.ServiceTags; import java.io.*; import java.net.MalformedURLException; import java.net.URL; +import java.nio.charset.Charset; import java.util.Map; public class RangerFileBasedTagRetriever extends RangerTagRetriever { @@ -37,6 +38,7 @@ public class RangerFileBasedTagRetriever extends RangerTagRetriever { private URL serviceTagsFileURL; private String serviceTagsFileName; + private Gson gsonBuilder; @Override public void init(Map<String, String> options) { @@ -44,6 +46,11 @@ public class RangerFileBasedTagRetriever extends RangerTagRetriever { if (LOG.isDebugEnabled()) { LOG.debug("==> init()" ); } + + gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") + .setPrettyPrinting() + .create(); + String serviceTagsFileNameProperty = "serviceTagsFileName"; String serviceTagsDefaultFileName = "/testdata/test_servicetags_hive.json"; @@ -99,9 +106,18 @@ public class RangerFileBasedTagRetriever extends RangerTagRetriever { } } + if (serviceTagsFileStream != null) { + try { + serviceTagsFileStream.close(); + } catch (Exception e) { + // Ignore + } + } + } else { LOG.error("FATAL: Cannot find service/serviceDef/serviceTagsFile to use for retrieving tags. Will NOT be able to retrieve tags."); } + if (LOG.isDebugEnabled()) { LOG.debug("<== init() : serviceTagsFileName=" + serviceTagsFileName); } @@ -117,13 +133,10 @@ public class RangerFileBasedTagRetriever extends RangerTagRetriever { ServiceTags serviceTags = null; if (serviceTagsFileURL != null) { - try { + try ( InputStream serviceTagsFileStream = serviceTagsFileURL.openStream(); - Reader reader = new InputStreamReader(serviceTagsFileStream); - - Gson gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") - .setPrettyPrinting() - .create(); + Reader reader = new InputStreamReader(serviceTagsFileStream, Charset.forName("UTF-8")) + ) { serviceTags = gsonBuilder.fromJson(reader, ServiceTags.class); @@ -131,7 +144,7 @@ public class RangerFileBasedTagRetriever extends RangerTagRetriever { // No change in serviceTags serviceTags = null; } - } catch (Exception e) { + } catch (IOException e) { LOG.warn("Error processing input file: or no privilege for reading file " + serviceTagsFileName); } } else { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fec84603/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractPredicateUtil.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractPredicateUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractPredicateUtil.java index 62b9bca..b154115 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractPredicateUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractPredicateUtil.java @@ -650,23 +650,24 @@ public class AbstractPredicateUtil { Map<String, RangerPolicyResource> policyResources = policy.getResources(); if(MapUtils.isNotEmpty(policyResources)) { - for(String resourceName : policyResources.keySet()) { - RangerPolicyResource policyResource = policyResources.get(resourceName); - if(policyResource != null && CollectionUtils.isNotEmpty(policyResource.getValues())) { - for(String policyResourceVal : policyResource.getValues()) { - if(StringUtils.containsIgnoreCase(policyResourceVal, resourceValue)) { + for (Map.Entry<String, RangerPolicyResource> entry : policyResources.entrySet()) { + + RangerPolicyResource policyResource = entry.getValue(); + + if (policyResource != null && CollectionUtils.isNotEmpty(policyResource.getValues())) { + + for (String policyResoureValue : policyResource.getValues()) { + if (StringUtils.containsIgnoreCase(policyResoureValue, resourceValue)) { ret = true; break; } } - } + } - if(ret) { - break; - } } + } } else { ret = true; @@ -806,11 +807,7 @@ public class AbstractPredicateUtil { return ret; } - - /** - * @param policySignature - * @return - */ + public Predicate createPredicateForResourceSignature(final String policySignature) { if (StringUtils.isEmpty(policySignature)) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fec84603/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java index 72da8e8..69781c5 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java @@ -89,7 +89,7 @@ public class PerfDataRecorder { } } - private class PerfStatistic { + private static class PerfStatistic { private AtomicLong numberOfInvocations = new AtomicLong(0L); private AtomicLong millisecondsSpent = new AtomicLong(0L); private AtomicLong minTimeSpent = new AtomicLong(Long.MAX_VALUE); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fec84603/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java index a45d71a..215d453 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java @@ -58,7 +58,7 @@ public class CommandLineParser // Instantiate a data-object and return ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount); } else { - showUsage(-1); + showUsage(); } return ret; } @@ -97,7 +97,8 @@ public class CommandLineParser CommandLine commandLine = commandLineParser.parse(options, args); if (commandLine.hasOption("h")) { - showUsage(0); + showUsage(); + return false; } servicePoliciesFileName = commandLine.getOptionValue("s"); @@ -264,12 +265,8 @@ public class CommandLineParser return ret; } - void showUsage(int exitCode) { + void showUsage() { HelpFormatter formater = new HelpFormatter(); formater.printHelp("perfTester", options); - - LOG.info("Exiting..."); - - System.exit(exitCode); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fec84603/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java index b88d670..e6095cb 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java @@ -25,12 +25,12 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.plugin.policyengine.*; -import java.io.FileNotFoundException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.lang.reflect.Type; import java.net.URL; +import java.nio.charset.Charset; import java.util.List; public class PerfTestClient extends Thread { @@ -84,7 +84,7 @@ public class PerfTestClient extends Thread { InputStream in = requestFileURL.openStream(); - reader = new InputStreamReader(in); + reader = new InputStreamReader(in, Charset.forName("UTF-8")); Type listType = new TypeToken<List<RequestData>>() { }.getType(); @@ -121,7 +121,7 @@ public class PerfTestClient extends Thread { try { for (int i = 0; i < maxCycles; i++) { for (RequestData data : requests) { - perfTestEngine.execute(data.request); + data.setResult(perfTestEngine.execute(data.getRequest())); } } } catch (Exception excp) { @@ -133,10 +133,28 @@ public class PerfTestClient extends Thread { } } - private class RequestData { - public String name; - public RangerAccessRequest request; - public RangerAccessResult result; + private static class RequestData { + private String name; + private RangerAccessRequest request; + private RangerAccessResult result; + + public RequestData() { + this(null, null, null); + } + + public RequestData(String name, RangerAccessRequest request, RangerAccessResult result) { + setName(name); + setRequest(request); + setResult(result); + } + + public String getName() {return name;} + public RangerAccessRequest getRequest() { return request;} + public RangerAccessResult getResult() { return result;} + + public void setName(String name) { this.name = name;} + public void setRequest(RangerAccessRequest request) { this.request = request;} + public void setResult(RangerAccessResult result) { this.result = result;} } static class RangerAccessRequestDeserializer implements JsonDeserializer<RangerAccessRequest> { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fec84603/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java index 1b5f6fd..38c6e2f 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java @@ -23,7 +23,6 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler; import org.apache.ranger.plugin.policyengine.*; import org.apache.ranger.plugin.util.ServicePolicies; @@ -31,6 +30,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.net.URL; +import java.nio.charset.Charset; public class PerfTestEngine { static final Log LOG = LogFactory.getLog(PerfTestEngine.class); @@ -60,7 +60,7 @@ public class PerfTestEngine { try { InputStream in = servicePoliciesFileURL.openStream(); - reader = new InputStreamReader(in); + reader = new InputStreamReader(in, Charset.forName("UTF-8")); servicePolicies = gsonBuilder.fromJson(reader, ServicePolicies.class); @@ -91,21 +91,23 @@ public class PerfTestEngine { return ret; } - public boolean execute(final RangerAccessRequest request) { + public RangerAccessResult execute(final RangerAccessRequest request) { if (LOG.isDebugEnabled()) { LOG.debug("==> execute(" + request + ")"); } - boolean ret = true; + RangerAccessResult ret = null; if (policyEvaluationEngine != null) { - RangerAccessResultProcessor auditHandler = null; - policyEvaluationEngine.preProcess(request); - RangerAccessResult result = policyEvaluationEngine.isAccessAllowed(request, auditHandler); + ret = policyEvaluationEngine.isAccessAllowed(request, null); + + if (LOG.isDebugEnabled()) { + LOG.debug("Executed request = {" + request + "}, result={" + ret + "}"); + } } else { LOG.error("Error executing request: PolicyEngine is null!"); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fec84603/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java index 28cc558..6c0d09a 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java @@ -24,11 +24,13 @@ import org.apache.commons.logging.LogFactory; import org.apache.ranger.plugin.util.PerfDataRecorder; import java.io.BufferedReader; +import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.URL; import java.nio.charset.Charset; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; public class RangerPolicyenginePerfTester { @@ -43,72 +45,77 @@ public class RangerPolicyenginePerfTester { PerfTestOptions perfTestOptions = commandLineParser.parse(args); - URL statCollectionFileURL = perfTestOptions.getStatCollectionFileURL(); + if (perfTestOptions != null) { + URL statCollectionFileURL = perfTestOptions.getStatCollectionFileURL(); - List<String> perfModuleNames = buildPerfModuleNames(statCollectionFileURL); + List<String> perfModuleNames = buildPerfModuleNames(statCollectionFileURL); - PerfDataRecorder.initialize(perfModuleNames); + PerfDataRecorder.initialize(perfModuleNames); - URL servicePoliciesFileURL = perfTestOptions.getServicePoliciesFileURL(); + URL servicePoliciesFileURL = perfTestOptions.getServicePoliciesFileURL(); - PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL); - if (!perfTestEngine.init()) { - LOG.error("Error initializing test data. Existing..."); - System.exit(1); - } + PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL); + if (!perfTestEngine.init()) { + LOG.error("Error initializing test data. Existing..."); + System.exit(1); + } - URL[] requestFileURLs = perfTestOptions.getRequestFileURLs(); - int requestFilesCount = requestFileURLs.length; + URL[] requestFileURLs = perfTestOptions.getRequestFileURLs(); + int requestFilesCount = requestFileURLs.length; - int clientsCount = perfTestOptions.getConcurrentClientCount(); - List<PerfTestClient> perfTestClients = new ArrayList<PerfTestClient>(clientsCount); + int clientsCount = perfTestOptions.getConcurrentClientCount(); + List<PerfTestClient> perfTestClients = new ArrayList<PerfTestClient>(clientsCount); - for (int i = 0; i < clientsCount; i++) { + for (int i = 0; i < clientsCount; i++) { - URL requestFileURL = requestFileURLs[i % requestFilesCount]; + URL requestFileURL = requestFileURLs[i % requestFilesCount]; - PerfTestClient perfTestClient = new PerfTestClient(perfTestEngine, i, requestFileURL, perfTestOptions.getIterationsCount()); + PerfTestClient perfTestClient = new PerfTestClient(perfTestEngine, i, requestFileURL, perfTestOptions.getIterationsCount()); - if (!perfTestClient.init()) { - LOG.error("Error initializing PerfTestClient: (id=" + i + ")"); - } else { - perfTestClients.add(perfTestClient); + if (!perfTestClient.init()) { + LOG.error("Error initializing PerfTestClient: (id=" + i + ")"); + } else { + perfTestClients.add(perfTestClient); + } } - } - if (LOG.isDebugEnabled()) { - LOG.debug("Number of perfTestClients=" + perfTestClients.size()); - } + if (LOG.isDebugEnabled()) { + LOG.debug("Number of perfTestClients=" + perfTestClients.size()); + } - for (PerfTestClient client : perfTestClients) { - try { - client.start(); - } catch (Throwable t) { - LOG.error("Error in starting client: " + client.getName(), t); + for (PerfTestClient client : perfTestClients) { + try { + client.start(); + } catch (Throwable t) { + LOG.error("Error in starting client: " + client.getName(), t); + } } - } - LOG.info("Waiting for " + perfTestClients.size() + " clients to finish up"); + LOG.info("Waiting for " + perfTestClients.size() + " clients to finish up"); - for (PerfTestClient client : perfTestClients) { - try { - if (client.isAlive()) { - LOG.info("Waiting for " + client.getName() + " to finish up."); - client.join(); + for (PerfTestClient client : perfTestClients) { + try { + if (client.isAlive()) { + LOG.info("Waiting for " + client.getName() + " to finish up."); + client.join(); + } + } catch (InterruptedException interruptedException) { + LOG.error("PerfTestClient.join() was interrupted"); } - } catch (InterruptedException interruptedException) { - LOG.error("PerfTestClient.join() was interrupted"); } - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== RangerPolicyenginePerfTester.main()"); - } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyenginePerfTester.main()"); + } - LOG.info("Completed performance-run"); + LOG.info("Completed performance-run"); - perfTestEngine.cleanup(); + perfTestEngine.cleanup(); + + PerfDataRecorder.getPerfDataRecorder().dumpStatistics(); + } + + LOG.info("Exiting..."); - PerfDataRecorder.getPerfDataRecorder().dumpStatistics(); } private static List<String> buildPerfModuleNames(URL statCollectionFileURL) { @@ -117,7 +124,7 @@ public class RangerPolicyenginePerfTester { try ( InputStream inStream = statCollectionFileURL.openStream(); InputStreamReader reader = new InputStreamReader(inStream, Charset.forName("UTF-8")); - BufferedReader br = new BufferedReader(reader); + BufferedReader br = new BufferedReader(reader) ) { String line; @@ -126,12 +133,10 @@ public class RangerPolicyenginePerfTester { line = line.trim(); if (!line.isEmpty() && !line.startsWith("#")) { String[] moduleNames = line.split(" "); - for (int i = 0; i < moduleNames.length; i++) { - perfModuleNames.add(moduleNames[i]); - } + perfModuleNames.addAll(Arrays.asList(moduleNames)); } } - } catch (Exception exception) { + } catch (IOException exception) { System.out.println("Error reading arguments:" + exception); }
