RANGER-2191: Update ranger-tool with new options to control Trie
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/383864c3 Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/383864c3 Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/383864c3 Branch: refs/heads/ranger-1.1 Commit: 383864c3ac5b0f6c013b39d04604d78895ead7bb Parents: fbcaaaf Author: Abhay Kulkarni <[email protected]> Authored: Thu Aug 16 23:53:39 2018 -0700 Committer: Mehul Parikh <[email protected]> Committed: Tue Sep 4 11:48:40 2018 +0530 ---------------------------------------------------------------------- .../RangerDefaultPolicyEvaluator.java | 4 +-- .../ranger/plugin/util/RangerResourceTrie.java | 20 +++++++------ ranger-tools/scripts/README.txt | 9 ++++-- .../ranger/policyengine/CommandLineParser.java | 30 ++++++++++++++++++-- .../ranger/policyengine/PerfTestOptions.java | 14 ++++++++- .../RangerPolicyenginePerfTester.java | 9 ++++++ ranger-tools/src/test/resources/commandline | 2 +- .../test/resources/testdata/ranger-config.xml | 18 ++++++++++++ ranger-tools/testdata/ranger-config.xml | 18 ++++++++++++ 9 files changed, 106 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java index 2a5ee54..289ec9b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java @@ -207,7 +207,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator @Override public void evaluate(RangerAccessRequest request, RangerAccessResult result) { if (LOG.isDebugEnabled()) { - LOG.debug("==> RangerDefaultPolicyEvaluator.evaluate(" + request + ", " + result + ")"); + LOG.debug("==> RangerDefaultPolicyEvaluator.evaluate(policyId=" + getPolicy().getId() + ", " + request + ", " + result + ")"); } RangerPerfTracer perf = null; @@ -257,7 +257,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator RangerPerfTracer.log(perf); if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerDefaultPolicyEvaluator.evaluate(" + request + ", " + result + ")"); + LOG.debug("<== RangerDefaultPolicyEvaluator.evaluate(policyId=" + getPolicy().getId() + ", " + request + ", " + result + ")"); } } http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java index e702684..9c7c775 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java @@ -68,7 +68,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> { RangerPerfTracer perf = null; if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) { - perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie(name=" + resourceDef.getName() + ")"); + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(name=" + resourceDef.getName() + ")"); } int builderThreadCount = RangerConfiguration.getInstance().getInt(TRIE_BUILDER_THREAD_COUNT, 1); @@ -154,6 +154,12 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> { LOG.debug("==> buildTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isMultiThreaded=" + (builderThreadCount > 1) + ")"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(resourceDef=" + resourceDef.getName() + ")"); + } + TrieNode<T> ret = new TrieNode<>(null); final boolean isMultiThreaded = builderThreadCount > 1; final List<ResourceTrieBuilderThread> builderThreads; @@ -243,7 +249,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> { RangerPerfTracer postSetupPerf = null; if (RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) { - postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie(name=" + resourceDef.getName() + "-postSetup)"); + postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(name=" + resourceDef.getName() + "-postSetup)"); } ret.postSetup(null, comparator); @@ -259,6 +265,8 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> { cleanUpThreads(builderThreads); } + RangerPerfTracer.logAlways(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== buildTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isMultiThreaded=" + isMultiThreaded + ") :" + ret); } @@ -324,11 +332,6 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> { } private void insert(TrieNode<T> currentRoot, String resource, boolean isRecursive, T evaluator) { - RangerPerfTracer perf = null; - - if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) { - perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.insert(resource=" + resource + ")"); - } TrieNode<T> curr = currentRoot; final String prefix = getNonWildcardPrefix(resource); @@ -344,7 +347,6 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> { curr.addEvaluator(evaluator); } - RangerPerfTracer.logAlways(perf); } private String getNonWildcardPrefix(String str) { @@ -559,7 +561,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> { RangerPerfTracer postSetupPerf = null; if (RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) { - postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie(thread=" + this.getName() + "-postSetup)"); + postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(thread=" + this.getName() + "-postSetup)"); } thisRoot.postSetup(parentWildcardEvaluators, comparator); http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/scripts/README.txt ---------------------------------------------------------------------- diff --git a/ranger-tools/scripts/README.txt b/ranger-tools/scripts/README.txt index 55170d5..81b5b66 100644 --- a/ranger-tools/scripts/README.txt +++ b/ranger-tools/scripts/README.txt @@ -51,6 +51,8 @@ This file describes how to build, setup, configure and run the performance testi tag-definitions, and service-resources with their associated tags; testdata/test_requests_hive.json - Contains access requests to be made to the policy-engine; + + testdata/ranger-config.xml - Contains any required Ranger configuration variables Please review the contents of these files and modify to suit your profiling needs. @@ -58,10 +60,13 @@ This file describes how to build, setup, configure and run the performance testi 6. Run the tool with the following command - % ./ranger-perftester.sh -s <service-policies-file> -r <requests-file> -c <number-of-concurrent-clients> -n <number-of-times-requests-file-to-be-run> + % ./ranger-perftester.sh -s <service-policies-file> -r <requests-file> -c <number-of-concurrent-clients> -n <number-of-times-requests-file-to-be-run> -t -d -f <ranger-configuration-file> -p <test-modules-file> + + where, -t indicates enabling Trie, + -d indicates enabling lazy post-setup of Trie structure, Example: - % ./ranger-perftester.sh -s testdata/test_servicepolicies_hive.json -r testdata/test_requests_hive.json -c 2 -n 1 + % ./ranger-perftester.sh -s testdata/test_servicepolicies_hive.json -r testdata/test_requests_hive.json -c 2 -n 1 -t -d -f testdata/ranger-config.xml -p testdata/test_modules.txt 7. At the end of the run, the performance-statistics are printed on the console and in the log specified file in conf/log4j.properties file as shown below. This is for time spent in evaluating access by Ranger Policy Engine during the course of a test run. The time values shown are in milliseconds. http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java index e8edd9e..fe9ec17 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java @@ -55,6 +55,12 @@ public class CommandLineParser private boolean isDynamicReorderingDisabled = true; private boolean isTrieLookupPrefixDisabled = true; + private boolean isLazyTriePostSetupDisabled = true; + + private String configurationFileName; + private URL configurationFileURL; + + private Options options = new Options(); CommandLineParser() {} @@ -63,7 +69,7 @@ public class CommandLineParser PerfTestOptions ret = null; if (parseArguments(args) && validateInputFiles()) { // Instantiate a data-object and return - ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount, isDynamicReorderingDisabled, isTrieLookupPrefixDisabled); + ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount, isDynamicReorderingDisabled, isTrieLookupPrefixDisabled, isLazyTriePostSetupDisabled, configurationFileURL); } else { showUsage(); } @@ -78,7 +84,10 @@ public class CommandLineParser -r request-file-name-list -n number-of-iterations -p modules-to-collect-stats + -f configuration-file-name -o + -t + -d If the concurrent-client-count is more than the number of files in the request-file-name-list, then reuse the request-file-names in a round-robin way @@ -98,8 +107,11 @@ public class CommandLineParser options.addOption("p", "statistics", true, "Modules for stat collection File Name"); options.addOption("c", "clients", true, "Number of concurrent clients"); options.addOption("n", "cycles", true, "Number of iterations"); + options.addOption("f", "configurations", true, "Configuration File Name"); options.addOption("o", "optimize", false, "Enable usage-based policy reordering"); options.addOption("t", "trie-prefilter", false, "Enable trie-prefilter"); + options.addOption("d", "trie-lazy-setup", false, "Enable lazy trie-setup"); + org.apache.commons.cli.CommandLineParser commandLineParser = new DefaultParser(); @@ -133,11 +145,20 @@ public class CommandLineParser isTrieLookupPrefixDisabled = false; } + if (commandLine.hasOption("d")) { + isLazyTriePostSetupDisabled = false; + } + + configurationFileName = commandLine.getOptionValue("f"); + if (LOG.isDebugEnabled()) { LOG.debug("servicePoliciesFileName=" + servicePoliciesFileName + ", requestFileName=" + Arrays.toString(requestFileNames)); LOG.debug("concurrentClientCount=" + concurrentClientCount + ", iterationsCount=" + iterationsCount); LOG.debug("isDynamicReorderingDisabled=" + isDynamicReorderingDisabled); LOG.debug("isTrieLookupPrefixDisabled=" + isTrieLookupPrefixDisabled); + LOG.debug("isLazyTriePostSetupDisabled=" + isLazyTriePostSetupDisabled); + LOG.debug("configurationFileName=" + configurationFileName); + } ret = true; @@ -164,11 +185,14 @@ public class CommandLineParser if (servicePoliciesFileURL != null) { if (requestFileNames != null) { if (validateRequestFiles()) { + ret = true; if (statCollectionFileName != null) { statCollectionFileURL = getInputFileURL(statCollectionFileName); ret = statCollectionFileURL != null; - } else { - ret = true; + } + if (ret && configurationFileName != null) { + configurationFileURL = getInputFileURL(configurationFileName); + ret = configurationFileURL != null; } } } else { http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java index d6e04ea..1486f52 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java @@ -28,12 +28,15 @@ public class PerfTestOptions { private final URL statCollectionFileURL; private final boolean isDynamicReorderingDisabled; private final boolean isTrieLookupPrefixDisabled; + private final boolean isOnDemandTriePostSetupDisabled; private final int concurrentClientCount; private final int iterationsCount; + private final URL perfConfigurationFileURL; - PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount, boolean isDynamicReorderingDisabled, boolean isTrieLookupPrefixDisabled) { + + PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount, boolean isDynamicReorderingDisabled, boolean isTrieLookupPrefixDisabled, boolean isOnDemandTriePostSetupDisabled, URL perfConfigurationFileURL) { this.servicePoliciesFileURL = servicePoliciesFileURL; this.requestFileURLs = requestFileURLs; this.statCollectionFileURL = statCollectionFileURL; @@ -41,6 +44,8 @@ public class PerfTestOptions { this.concurrentClientCount = concurrentClientCount; this.isDynamicReorderingDisabled = isDynamicReorderingDisabled; this.isTrieLookupPrefixDisabled = isTrieLookupPrefixDisabled; + this.isOnDemandTriePostSetupDisabled = isOnDemandTriePostSetupDisabled; + this.perfConfigurationFileURL = perfConfigurationFileURL; } public URL getServicePoliciesFileURL() { @@ -66,4 +71,11 @@ public class PerfTestOptions { public boolean getIsDynamicReorderingDisabled() { return isDynamicReorderingDisabled; } public boolean getIsTrieLookupPrefixDisabled() { return isTrieLookupPrefixDisabled; } + + public boolean getIsOnDemandTriePostSetupDisabled() { return isOnDemandTriePostSetupDisabled; } + + public URL getPerfConfigurationFileURL() { + return this.perfConfigurationFileURL; + } + } http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java index 78cbe02..53e5cf0 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java @@ -21,6 +21,7 @@ package org.apache.ranger.policyengine; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; import org.apache.ranger.plugin.util.PerfDataRecorder; @@ -61,6 +62,14 @@ public class RangerPolicyenginePerfTester { policyEngineOptions.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; policyEngineOptions.cacheAuditResults = false; policyEngineOptions.disableTrieLookupPrefilter = perfTestOptions.getIsTrieLookupPrefixDisabled(); + policyEngineOptions.optimizeTrieForRetrieval = perfTestOptions.getIsOnDemandTriePostSetupDisabled(); + + URL configurationFileURL = perfTestOptions.getPerfConfigurationFileURL(); + + if (configurationFileURL != null) { + RangerConfiguration config = RangerConfiguration.getInstance(); + config.addResource(configurationFileURL); + } PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL, policyEngineOptions, perfTestOptions.getIsDynamicReorderingDisabled()); if (!perfTestEngine.init()) { http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/src/test/resources/commandline ---------------------------------------------------------------------- diff --git a/ranger-tools/src/test/resources/commandline b/ranger-tools/src/test/resources/commandline index 9ea690e..73338d5 100644 --- a/ranger-tools/src/test/resources/commandline +++ b/ranger-tools/src/test/resources/commandline @@ -17,4 +17,4 @@ # under the License. # --s /testdata/test_servicepolicies_hive.json -r /testdata/test_requests_hive.json -p /testdata/test_modules.txt -c 3 -n 1 +-s /testdata/test_servicepolicies_hive.json -r /testdata/test_requests_hive.json -p /testdata/test_modules.txt -c 3 -n 1 -t -d -f /testdata/ranger-config.xml http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/src/test/resources/testdata/ranger-config.xml ---------------------------------------------------------------------- diff --git a/ranger-tools/src/test/resources/testdata/ranger-config.xml b/ranger-tools/src/test/resources/testdata/ranger-config.xml new file mode 100644 index 0000000..933d670 --- /dev/null +++ b/ranger-tools/src/test/resources/testdata/ranger-config.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> + +<configuration> + <property> + <name>ranger.policyengine.trie.builder.thread.count</name> + <value>1</value> + </property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/testdata/ranger-config.xml ---------------------------------------------------------------------- diff --git a/ranger-tools/testdata/ranger-config.xml b/ranger-tools/testdata/ranger-config.xml new file mode 100644 index 0000000..933d670 --- /dev/null +++ b/ranger-tools/testdata/ranger-config.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> + +<configuration> + <property> + <name>ranger.policyengine.trie.builder.thread.count</name> + <value>1</value> + </property> +</configuration> \ No newline at end of file
