RANGER-794: commit id fc5314e8b79e9d754c12a63bb67a0f0190ddfe9f Signed-off-by: Madhan Neethiraj <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/36fbb78f Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/36fbb78f Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/36fbb78f Branch: refs/heads/ranger-0.5 Commit: 36fbb78f0bb41e4a1f3e25c050b079f09ed4f668 Parents: bec2fef Author: Abhay Kulkarni <[email protected]> Authored: Mon Jan 18 15:31:34 2016 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Sun Mar 20 10:34:46 2016 -0700 ---------------------------------------------------------------------- .../ranger/plugin/util/PerfDataRecorder.java | 58 ++++++++++++++++---- .../plugin/util/RangerPerfCollectorTracer.java | 6 +- .../plugin/util/RangerPerfTracerFactory.java | 6 +- ranger-tools/conf/log4j.properties | 11 +--- ranger-tools/scripts/README.txt | 21 +++---- .../ranger/policyengine/CommandLineParser.java | 4 +- .../RangerPolicyenginePerfTester.java | 29 +++++++--- ranger-tools/testdata/test_modules.txt | 25 +++++++++ 8 files changed, 111 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/36fbb78f/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java index 72da8e8..9b29075 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java @@ -23,6 +23,8 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,25 +34,43 @@ public class PerfDataRecorder { private static final Log LOG = LogFactory.getLog(PerfDataRecorder.class); private static final Log PERF = RangerPerfTracer.getPerfLogger(PerfDataRecorder.class); - static PerfDataRecorder instance = null; + private static volatile PerfDataRecorder instance = null; private Map<String, PerfStatistic> perfStatistics = new HashMap<String, PerfStatistic>(); + private boolean initPerfStatisticsOnce = true; public static void initialize(List<String> names) { - if (getPerfDataRecorder() == null) { - instance = new PerfDataRecorder(); + if (instance == null) { + synchronized (PerfDataRecorder.class) { + if (instance == null) { + instance = new PerfDataRecorder(names); + } + } } - instance.init(names); } - public static PerfDataRecorder getPerfDataRecorder() { - return instance; + public static boolean collectStatistics() { + return instance != null; + } + + public static void printStatistics() { + if (instance != null) { + instance.dumpStatistics(); + } } + public static void recordStatistic(String tag, long elapsedTime) { + if (instance != null) { + instance.record(tag, elapsedTime); + } + } + + private void dumpStatistics() { + List<String> tags = new ArrayList<String>(perfStatistics.keySet()); - public void dumpStatistics() { - for (Map.Entry<String, PerfStatistic> entry : perfStatistics.entrySet()) { + Collections.sort(tags); - String tag = entry.getKey(); - PerfStatistic perfStatistic = entry.getValue(); + for (String tag : tags) { + + PerfStatistic perfStatistic = perfStatistics.get(tag); long averageTimeSpent = 0L; long minTimeSpent = 0L; @@ -73,19 +93,33 @@ public class PerfDataRecorder { } } - void record(String tag, long elapsedTime) { + private void record(String tag, long elapsedTime) { PerfStatistic perfStatistic = perfStatistics.get(tag); + + if (perfStatistic == null && !initPerfStatisticsOnce) { + synchronized (PerfDataRecorder.class) { + perfStatistic = perfStatistics.get(tag); + if (perfStatistic == null) { + perfStatistic = new PerfStatistic(); + perfStatistics.put(tag, perfStatistic); + } + } + } + if (perfStatistic != null) { perfStatistic.addPerfDataItem(elapsedTime); } + } - private void init(List<String> names) { + private PerfDataRecorder(List<String> names) { if (CollectionUtils.isNotEmpty(names)) { for (String name : names) { // Create structure perfStatistics.put(name, new PerfStatistic()); } + } else { + initPerfStatisticsOnce = false; } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/36fbb78f/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java index d092859..d899c6f 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java @@ -22,15 +22,13 @@ package org.apache.ranger.plugin.util; import org.apache.commons.logging.Log; public class RangerPerfCollectorTracer extends RangerPerfTracer { - private final PerfDataRecorder recorder; - public RangerPerfCollectorTracer(Log logger, String tag, String data, PerfDataRecorder recorder) { + public RangerPerfCollectorTracer(Log logger, String tag, String data) { super(logger, tag, data); - this.recorder = recorder; } @Override public void log() { - recorder.record(tag, getElapsedTime()); + PerfDataRecorder.recordStatistic(tag, getElapsedTime()); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/36fbb78f/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java index 8db2d45..1153091 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java @@ -23,14 +23,12 @@ import org.apache.commons.logging.Log; public class RangerPerfTracerFactory { - private static PerfDataRecorder perfDataRecorder = PerfDataRecorder.getPerfDataRecorder(); - static RangerPerfTracer getPerfTracer(Log logger, String tag, String data) { RangerPerfTracer ret = null; - if (perfDataRecorder != null) { - ret = new RangerPerfCollectorTracer(logger, tag, data, perfDataRecorder); + if (PerfDataRecorder.collectStatistics()) { + ret = new RangerPerfCollectorTracer(logger, tag, data); } else if (logger.isDebugEnabled()) { ret = new RangerPerfTracer(logger, tag, data); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/36fbb78f/ranger-tools/conf/log4j.properties ---------------------------------------------------------------------- diff --git a/ranger-tools/conf/log4j.properties b/ranger-tools/conf/log4j.properties index 86f5c18..21f7fad 100644 --- a/ranger-tools/conf/log4j.properties +++ b/ranger-tools/conf/log4j.properties @@ -13,11 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -##-- To prevent junits from cluttering the build run by default all test runs send output to null appender -log4j.appender.devnull=org.apache.log4j.varia.NullAppender -# ranger.root.logger=FATAL,devnull - -##-- uncomment the following line during during development/debugging so see debug messages during test run to be emitted to console ranger.root.logger=INFO,console log4j.rootLogger=${ranger.root.logger} @@ -25,14 +20,10 @@ log4j.rootLogger=${ranger.root.logger} # Logging Threshold log4j.threshold=ALL -# -# console -# Add "console" to rootlogger above if you want to use this -# log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console.target=System.err log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %L %m%n +log4j.appender.console.layout.ConversionPattern=%m%n # # ranger.perf log level http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/36fbb78f/ranger-tools/scripts/README.txt ---------------------------------------------------------------------- diff --git a/ranger-tools/scripts/README.txt b/ranger-tools/scripts/README.txt index dda6fd1..53a3a8b 100644 --- a/ranger-tools/scripts/README.txt +++ b/ranger-tools/scripts/README.txt @@ -37,28 +37,29 @@ This file describes how to build, setup, configure and run the performance testi ranger-0.5.0-ranger-tools/conf ranger-0.5.0-ranger-tools/dist ranger-0.5.0-ranger-tools/lib - ranger-0.5.0-ranger-tools/scripts - ranger-0.5.0-ranger-tools/testdata 4. % cd ranger-0.5.0-ranger-tools 5. Configure the policies and requests to use in the test run Following sample data files are packaged with the perf-tool: - service-policies - testdata/test_servicepolicies_hive.json - requests - testdata/test_requests_hive.json - modules-to-profile - testdata/test_modules.txt - Please review the contents of these files and modify (or copy/modify) to suite your policy and request needs. + testdata/test_servicepolicies_hive.json - Contains service-policies used to initialize the policy-engine; - Update conf/log4j.properties to specify the filename where perf run results will be written to. Property to update is 'ranger.perf.logger'. + testdata/test_requests_hive.json - Contains access requests to be made to the policy-engine; + + Please review the contents of these files and modify to suit your profiling needs. + + Update conf/log4j.properties to specify the filename where perf run results will be written to. Property to update is 'log4j.appender.PERF.File'. 6. Run the tool with the following command - % ./ranger-perftester.sh -s <service-policies-file> -r <requests-file> -p <profiled-modules-file> -c <number-of-concurrent-clients> -n <number-of-times-requests-file-to-be-run> + % ./ranger-perftester.sh -s <service-policies-file> -r <requests-file> -c <number-of-concurrent-clients> -n <number-of-times-requests-file-to-be-run> Example: - % ./ranger-perftester.sh -s testdata/test_servicepolicies_hive.json -r testdata/test_requests_hive.json -p testdata/test_modules.txt -c 2 -n 1 + % ./ranger-perftester.sh -s testdata/test_servicepolicies_hive.json -r testdata/test_requests_hive.json -c 2 -n 1 + +7. At the end of the run, the performance-statistics are printed on the console and in the log specified file in conf/log4j.properties file as shown below. This is for time spent in evaluating access by Ranger Policy Engine during the course of a test run. The time values shown are in milliseconds. -7. At the end of the run, the performance-statistics are printed on the console and in the log specified file in conf/log4j.properties file. +[RangerPolicyEngine.isAccessAllowed] execCount:64, totalTimeTaken:1873, maxTimeTaken:276, minTimeTaken:4, avgTimeTaken:29 http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/36fbb78f/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java index a45d71a..0dc79a0 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java @@ -147,8 +147,8 @@ public class CommandLineParser if (statCollectionFileName != null) { statCollectionFileURL = getInputFileURL(statCollectionFileName); ret = statCollectionFileURL != null; - } else { - LOG.error("Error processing stat-collection-module file"); + } else { + ret = true; } } } else { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/36fbb78f/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java index 28cc558..bcd1c68 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java @@ -45,7 +45,7 @@ public class RangerPolicyenginePerfTester { URL statCollectionFileURL = perfTestOptions.getStatCollectionFileURL(); - List<String> perfModuleNames = buildPerfModuleNames(statCollectionFileURL); + List<String> perfModuleNames = statCollectionFileURL != null ? buildPerfModuleNames(statCollectionFileURL) : new ArrayList<String>(); PerfDataRecorder.initialize(perfModuleNames); @@ -108,17 +108,19 @@ public class RangerPolicyenginePerfTester { perfTestEngine.cleanup(); - PerfDataRecorder.getPerfDataRecorder().dumpStatistics(); + PerfDataRecorder.printStatistics(); } private static List<String> buildPerfModuleNames(URL statCollectionFileURL) { List<String> perfModuleNames = new ArrayList<String>(); - try ( - InputStream inStream = statCollectionFileURL.openStream(); - InputStreamReader reader = new InputStreamReader(inStream, Charset.forName("UTF-8")); - BufferedReader br = new BufferedReader(reader); - ) { + InputStream inStream = null; + InputStreamReader reader = null; + BufferedReader br = null; + try { + inStream = statCollectionFileURL.openStream(); + reader = new InputStreamReader(inStream, Charset.forName("UTF-8")); + br = new BufferedReader(reader); String line; @@ -132,6 +134,19 @@ public class RangerPolicyenginePerfTester { } } } catch (Exception exception) { + try { + if (br != null) { + br.close(); + } + if (reader != null) { + reader.close(); + } + if (inStream != null) { + inStream.close(); + } + } catch (Exception e) { + // Ignore + } System.out.println("Error reading arguments:" + exception); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/36fbb78f/ranger-tools/testdata/test_modules.txt ---------------------------------------------------------------------- diff --git a/ranger-tools/testdata/test_modules.txt b/ranger-tools/testdata/test_modules.txt index 9ffcbfc..f317aaf 100644 --- a/ranger-tools/testdata/test_modules.txt +++ b/ranger-tools/testdata/test_modules.txt @@ -19,6 +19,7 @@ PolicyRefresher.loadPolicy RangerPolicyEngine.init +RangerPolicyEngine.cleanUp RangerContextEnricher.init RangerPolicyEvaluator.init RangerPolicyItemEvaluator.init @@ -34,3 +35,27 @@ RangerTagRefresher.populateTags RangerPolicyEvaluator.isAccessAllowed RangerPolicyRetriever.getServicePolicies RangerTagDBReceiver.getTags +ServiceREST.createServiceDef +ServiceREST.updateServiceDef +ServiceREST.deleteServiceDef +ServiceREST.getServiceDef +ServiceREST.getServiceDefByName +ServiceREST.getServiceDefs +ServiceREST.createService +ServiceREST.updateService +ServiceREST.deleteService +ServiceREST.getService +ServiceREST.getServices +ServiceREST.countService +ServiceREST.validateConfig +ServiceREST.lookupResource +ServiceREST.grantAccess +ServiceREST.revokeAccess +ServiceREST.createPolicy +ServiceREST.updatePolicy +ServiceREST.deletePolicy +ServiceREST.getPolicy +ServiceREST.getPolicies +ServiceREST.countPolicies +ServiceREST.getServicePolicies +ServiceREST.getServicePoliciesIfUpdated \ No newline at end of file
