Repository: incubator-ranger Updated Branches: refs/heads/master f79bc59a0 -> fc5314e8b
RANGER-794: misc updates to performance measurement instrumentation: uses fewer command-line arguments, display the result in sorted order Signed-off-by: Madhan Neethiraj <[email protected]> sorted output Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/fc5314e8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/fc5314e8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/fc5314e8 Branch: refs/heads/master Commit: fc5314e8b79e9d754c12a63bb67a0f0190ddfe9f Parents: f79bc59 Author: Abhay Kulkarni <[email protected]> Authored: Mon Jan 18 15:31:34 2016 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Tue Jan 19 12:45:08 2016 -0800 ---------------------------------------------------------------------- .../policyengine/RangerPolicyRepository.java | 2 +- .../ranger/plugin/util/PerfDataRecorder.java | 58 ++++++++++++++++---- .../plugin/util/RangerPerfCollectorTracer.java | 7 +-- .../plugin/util/RangerPerfTracerFactory.java | 6 +- agents-common/src/test/resources/log4j.xml | 5 ++ ranger-tools/conf/log4j.properties | 11 +--- ranger-tools/scripts/README.txt | 26 +++++---- .../ranger/policyengine/CommandLineParser.java | 4 +- .../RangerPolicyenginePerfTester.java | 4 +- ranger-tools/testdata/test_modules.txt | 25 +++++++++ 10 files changed, 101 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fc5314e8/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index ee36d34..641320f 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -327,7 +327,7 @@ public class RangerPolicyRepository { RangerPerfTracer perf = null; if(RangerPerfTracer.isPerfTraceEnabled(PERF_CONTEXTENRICHER_INIT_LOG)) { - perf = RangerPerfTracer.getPerfTracer(PERF_CONTEXTENRICHER_INIT_LOG, "RangerContextEnricher.init(appId=", appId + ",name=" + enricherDef.getName() + ")"); + perf = RangerPerfTracer.getPerfTracer(PERF_CONTEXTENRICHER_INIT_LOG, "RangerContextEnricher.init(appId=" + appId + ",name=" + enricherDef.getName() + ")"); } String name = enricherDef != null ? enricherDef.getName() : null; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fc5314e8/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java index 69781c5..ddc42c0 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java @@ -23,6 +23,8 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,25 +34,43 @@ public class PerfDataRecorder { private static final Log LOG = LogFactory.getLog(PerfDataRecorder.class); private static final Log PERF = RangerPerfTracer.getPerfLogger(PerfDataRecorder.class); - static PerfDataRecorder instance = null; + private static volatile PerfDataRecorder instance = null; private Map<String, PerfStatistic> perfStatistics = new HashMap<String, PerfStatistic>(); + private boolean initPerfStatisticsOnce = true; public static void initialize(List<String> names) { - if (getPerfDataRecorder() == null) { - instance = new PerfDataRecorder(); + if (instance == null) { + synchronized (PerfDataRecorder.class) { + if (instance == null) { + instance = new PerfDataRecorder(names); + } + } } - instance.init(names); } - public static PerfDataRecorder getPerfDataRecorder() { - return instance; + public static boolean collectStatistics() { + return instance != null; + } + + public static void printStatistics() { + if (instance != null) { + instance.dumpStatistics(); + } } + public static void recordStatistic(String tag, long elapsedTime) { + if (instance != null) { + instance.record(tag, elapsedTime); + } + } + + private void dumpStatistics() { + List<String> tags = new ArrayList<String>(perfStatistics.keySet()); - public void dumpStatistics() { - for (Map.Entry<String, PerfStatistic> entry : perfStatistics.entrySet()) { + Collections.sort(tags); - String tag = entry.getKey(); - PerfStatistic perfStatistic = entry.getValue(); + for (String tag : tags) { + + PerfStatistic perfStatistic = perfStatistics.get(tag); long averageTimeSpent = 0L; long minTimeSpent = 0L; @@ -73,19 +93,33 @@ public class PerfDataRecorder { } } - void record(String tag, long elapsedTime) { + private void record(String tag, long elapsedTime) { PerfStatistic perfStatistic = perfStatistics.get(tag); + + if (perfStatistic == null && !initPerfStatisticsOnce) { + synchronized (PerfDataRecorder.class) { + perfStatistic = perfStatistics.get(tag); + if (perfStatistic == null) { + perfStatistic = new PerfStatistic(); + perfStatistics.put(tag, perfStatistic); + } + } + } + if (perfStatistic != null) { perfStatistic.addPerfDataItem(elapsedTime); } + } - private void init(List<String> names) { + private PerfDataRecorder(List<String> names) { if (CollectionUtils.isNotEmpty(names)) { for (String name : names) { // Create structure perfStatistics.put(name, new PerfStatistic()); } + } else { + initPerfStatisticsOnce = false; } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fc5314e8/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java index e7b3865..d899c6f 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java @@ -19,19 +19,16 @@ package org.apache.ranger.plugin.util; -import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; public class RangerPerfCollectorTracer extends RangerPerfTracer { - private final PerfDataRecorder recorder; - public RangerPerfCollectorTracer(Log logger, String tag, String data, PerfDataRecorder recorder) { + public RangerPerfCollectorTracer(Log logger, String tag, String data) { super(logger, tag, data); - this.recorder = recorder; } @Override public void log() { - recorder.record(tag, getElapsedTime()); + PerfDataRecorder.recordStatistic(tag, getElapsedTime()); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fc5314e8/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java index 8db2d45..1153091 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java @@ -23,14 +23,12 @@ import org.apache.commons.logging.Log; public class RangerPerfTracerFactory { - private static PerfDataRecorder perfDataRecorder = PerfDataRecorder.getPerfDataRecorder(); - static RangerPerfTracer getPerfTracer(Log logger, String tag, String data) { RangerPerfTracer ret = null; - if (perfDataRecorder != null) { - ret = new RangerPerfCollectorTracer(logger, tag, data, perfDataRecorder); + if (PerfDataRecorder.collectStatistics()) { + ret = new RangerPerfCollectorTracer(logger, tag, data); } else if (logger.isDebugEnabled()) { ret = new RangerPerfTracer(logger, tag, data); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fc5314e8/agents-common/src/test/resources/log4j.xml ---------------------------------------------------------------------- diff --git a/agents-common/src/test/resources/log4j.xml b/agents-common/src/test/resources/log4j.xml index 1f2f78c..f9a613b 100644 --- a/agents-common/src/test/resources/log4j.xml +++ b/agents-common/src/test/resources/log4j.xml @@ -48,6 +48,11 @@ <level value="debug" /> <appender-ref ref="ranger_perf_appender" /> </logger> + + <logger name="org.apache.ranger.perf.rest.ServiceREST" additivity="false"> + <level value="debug" /> + <appender-ref ref="ranger_perf_appender" /> + </logger> --> <root> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fc5314e8/ranger-tools/conf/log4j.properties ---------------------------------------------------------------------- diff --git a/ranger-tools/conf/log4j.properties b/ranger-tools/conf/log4j.properties index 86f5c18..21f7fad 100644 --- a/ranger-tools/conf/log4j.properties +++ b/ranger-tools/conf/log4j.properties @@ -13,11 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -##-- To prevent junits from cluttering the build run by default all test runs send output to null appender -log4j.appender.devnull=org.apache.log4j.varia.NullAppender -# ranger.root.logger=FATAL,devnull - -##-- uncomment the following line during during development/debugging so see debug messages during test run to be emitted to console ranger.root.logger=INFO,console log4j.rootLogger=${ranger.root.logger} @@ -25,14 +20,10 @@ log4j.rootLogger=${ranger.root.logger} # Logging Threshold log4j.threshold=ALL -# -# console -# Add "console" to rootlogger above if you want to use this -# log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console.target=System.err log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %L %m%n +log4j.appender.console.layout.ConversionPattern=%m%n # # ranger.perf log level http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fc5314e8/ranger-tools/scripts/README.txt ---------------------------------------------------------------------- diff --git a/ranger-tools/scripts/README.txt b/ranger-tools/scripts/README.txt index 0469a6f..7dc84e9 100644 --- a/ranger-tools/scripts/README.txt +++ b/ranger-tools/scripts/README.txt @@ -37,29 +37,33 @@ This file describes how to build, setup, configure and run the performance testi ranger-0.5.0-ranger-tools/conf ranger-0.5.0-ranger-tools/dist ranger-0.5.0-ranger-tools/lib - ranger-0.5.0-ranger-tools/scripts - ranger-0.5.0-ranger-tools/testdata 4. % cd ranger-0.5.0-ranger-tools 5. Configure the policies and requests to use in the test run Following sample data files are packaged with the perf-tool: - service-policies - testdata/test_servicepolicies_hive.json - requests - testdata/test_requests_hive.json - modules-to-profile - testdata/test_modules.txt - service-tags - testdata/test_servicetags_hive.json (used only for tag-based policies; referenced from service-policies file. - Please review the contents of these files and modify (or copy/modify) to suite your policy and request needs. + testdata/test_servicepolicies_hive.json - Contains service-policies used to initialize the policy-engine; - Update conf/log4j.properties to specify the filename where perf run results will be written to. Property to update is 'ranger.perf.logger'. + testdata/test_servicetags_hive.json - This is used only for tag-based policies. This is referenced + from service-policies file. It contains specification of + tag-definitions, and service-resources with their associated tags; + + testdata/test_requests_hive.json - Contains access requests to be made to the policy-engine; + + Please review the contents of these files and modify to suit your profiling needs. + + Update conf/log4j.properties to specify the filename where perf run results will be written to. Property to update is 'log4j.appender.PERF.File'. 6. Run the tool with the following command - % ./ranger-perftester.sh -s <service-policies-file> -r <requests-file> -p <profiled-modules-file> -c <number-of-concurrent-clients> -n <number-of-times-requests-file-to-be-run> + % ./ranger-perftester.sh -s <service-policies-file> -r <requests-file> -c <number-of-concurrent-clients> -n <number-of-times-requests-file-to-be-run> Example: - % ./ranger-perftester.sh -s testdata/test_servicepolicies_hive.json -r testdata/test_requests_hive.json -p testdata/test_modules.txt -c 2 -n 1 + % ./ranger-perftester.sh -s testdata/test_servicepolicies_hive.json -r testdata/test_requests_hive.json -c 2 -n 1 + +7. At the end of the run, the performance-statistics are printed on the console and in the log specified file in conf/log4j.properties file as shown below. This is for time spent in evaluating access by Ranger Policy Engine during the course of a test run. The time values shown are in milliseconds. -7. At the end of the run, the performance-statistics are printed on the console and in the log specified file in conf/log4j.properties file. +[RangerPolicyEngine.isAccessAllowed] execCount:64, totalTimeTaken:1873, maxTimeTaken:276, minTimeTaken:4, avgTimeTaken:29 http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fc5314e8/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java index 215d453..abc88d7 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java @@ -148,8 +148,8 @@ public class CommandLineParser if (statCollectionFileName != null) { statCollectionFileURL = getInputFileURL(statCollectionFileName); ret = statCollectionFileURL != null; - } else { - LOG.error("Error processing stat-collection-module file"); + } else { + ret = true; } } } else { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fc5314e8/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java index 6c0d09a..42d7cde 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java @@ -48,7 +48,7 @@ public class RangerPolicyenginePerfTester { if (perfTestOptions != null) { URL statCollectionFileURL = perfTestOptions.getStatCollectionFileURL(); - List<String> perfModuleNames = buildPerfModuleNames(statCollectionFileURL); + List<String> perfModuleNames = statCollectionFileURL != null ? buildPerfModuleNames(statCollectionFileURL) : new ArrayList<String>(); PerfDataRecorder.initialize(perfModuleNames); @@ -111,7 +111,7 @@ public class RangerPolicyenginePerfTester { perfTestEngine.cleanup(); - PerfDataRecorder.getPerfDataRecorder().dumpStatistics(); + PerfDataRecorder.printStatistics(); } LOG.info("Exiting..."); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fc5314e8/ranger-tools/testdata/test_modules.txt ---------------------------------------------------------------------- diff --git a/ranger-tools/testdata/test_modules.txt b/ranger-tools/testdata/test_modules.txt index 9ffcbfc..f317aaf 100644 --- a/ranger-tools/testdata/test_modules.txt +++ b/ranger-tools/testdata/test_modules.txt @@ -19,6 +19,7 @@ PolicyRefresher.loadPolicy RangerPolicyEngine.init +RangerPolicyEngine.cleanUp RangerContextEnricher.init RangerPolicyEvaluator.init RangerPolicyItemEvaluator.init @@ -34,3 +35,27 @@ RangerTagRefresher.populateTags RangerPolicyEvaluator.isAccessAllowed RangerPolicyRetriever.getServicePolicies RangerTagDBReceiver.getTags +ServiceREST.createServiceDef +ServiceREST.updateServiceDef +ServiceREST.deleteServiceDef +ServiceREST.getServiceDef +ServiceREST.getServiceDefByName +ServiceREST.getServiceDefs +ServiceREST.createService +ServiceREST.updateService +ServiceREST.deleteService +ServiceREST.getService +ServiceREST.getServices +ServiceREST.countService +ServiceREST.validateConfig +ServiceREST.lookupResource +ServiceREST.grantAccess +ServiceREST.revokeAccess +ServiceREST.createPolicy +ServiceREST.updatePolicy +ServiceREST.deletePolicy +ServiceREST.getPolicy +ServiceREST.getPolicies +ServiceREST.countPolicies +ServiceREST.getServicePolicies +ServiceREST.getServicePoliciesIfUpdated \ No newline at end of file
