RANGER-794: Ranger policy engine performance measurement
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/fbf4f353 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/fbf4f353 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/fbf4f353 Branch: refs/heads/master Commit: fbf4f3533d0c39d018d2ac92538f77761ca461d3 Parents: ea559d3 Author: Abhay Kulkarni <[email protected]> Authored: Mon Dec 21 17:53:51 2015 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Mon Dec 21 17:53:51 2015 -0800 ---------------------------------------------------------------------- .../RangerFileBasedTagRetriever.java | 149 + .../contextenricher/RangerTagEnricher.java | 9 +- .../ranger/plugin/util/PerfDataRecorder.java | 113 + .../plugin/util/RangerPerfCollectorTracer.java | 37 + .../ranger/plugin/util/RangerPerfTracer.java | 34 +- .../plugin/util/RangerPerfTracerFactory.java | 39 + pom.xml | 3 + ranger-tools/.gitignore | 2 + ranger-tools/conf/log4j.properties | 50 + ranger-tools/pom.xml | 69 + ranger-tools/scripts/README.txt | 65 + ranger-tools/scripts/ranger-perftester.sh | 32 + ranger-tools/scripts/summary.awk | 17 + .../ranger/policyengine/CommandLineParser.java | 275 + .../ranger/policyengine/PerfTestClient.java | 161 + .../ranger/policyengine/PerfTestEngine.java | 126 + .../ranger/policyengine/PerfTestOptions.java | 60 + .../RangerPolicyenginePerfTester.java | 140 + .../ranger/policyengine/PerfTesterTest.java | 112 + ranger-tools/src/test/resources/commandline | 20 + .../src/test/resources/log4j.properties | 50 + .../test/resources/testdata/test_modules.txt | 24 + .../resources/testdata/test_requests_hive.json | 10 + .../testdata/test_servicepolicies_hive.json | 488 + .../testdata/test_servicetags_hive.json | 62 + ranger-tools/testdata/test_modules.txt | 23 + ranger-tools/testdata/test_requests_hive.json | 257 + .../testdata/test_servicepolicies_hive.json | 252899 ++++++++++++++++ .../testdata/test_servicetags_hive.json | 23966 ++ src/main/assembly/ranger-tools.xml | 134 + 30 files changed, 279413 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java new file mode 100644 index 0000000..95291be --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.contextenricher; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.util.ServiceTags; + +import java.io.*; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Map; + +public class RangerFileBasedTagRetriever extends RangerTagRetriever { + private static final Log LOG = LogFactory.getLog(RangerFileBasedTagRetriever.class); + + + private URL serviceTagsFileURL; + private String serviceTagsFileName; + + @Override + public void init(Map<String, String> options) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> init()" ); + } + String serviceTagsFileNameProperty = "serviceTagsFileName"; + String serviceTagsDefaultFileName = "/testdata/test_servicetags_hive.json"; + + if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId)) { + InputStream serviceTagsFileStream = null; + + + // Open specified file from options- it should contain service-tags + + serviceTagsFileName = options != null? options.get(serviceTagsFileNameProperty) : null; + + serviceTagsFileName = serviceTagsFileName == null ? serviceTagsDefaultFileName : serviceTagsFileName; + + File f = new File(serviceTagsFileName); + + if (f.exists() && f.isFile() && f.canRead()) { + try { + serviceTagsFileStream = new FileInputStream(f); + serviceTagsFileURL = f.toURI().toURL(); + } catch (FileNotFoundException exception) { + LOG.error("Error processing input file:" + serviceTagsFileName + " or no privilege for reading file " + serviceTagsFileName, exception); + } catch (MalformedURLException malformedException) { + LOG.error("Error processing input file:" + serviceTagsFileName + " cannot be converted to URL " + serviceTagsFileName, malformedException); + } + } else { + + URL fileURL = getClass().getResource(serviceTagsFileName); + if (fileURL == null) { + if (!serviceTagsFileName.startsWith("/")) { + fileURL = getClass().getResource("/" + serviceTagsFileName); + } + } + + if (fileURL == null) { + fileURL = ClassLoader.getSystemClassLoader().getResource(serviceTagsFileName); + if (fileURL == null) { + if (!serviceTagsFileName.startsWith("/")) { + fileURL = ClassLoader.getSystemClassLoader().getResource("/" + serviceTagsFileName); + } + } + } + + if (fileURL != null) { + + try { + serviceTagsFileStream = fileURL.openStream(); + serviceTagsFileURL = fileURL; + } catch (Exception exception) { + LOG.error(serviceTagsFileName + " is not a file", exception); + } + } else { + LOG.warn("Error processing input file: URL not found for " + serviceTagsFileName + " or no privilege for reading file " + serviceTagsFileName); + } + } + + } else { + LOG.error("FATAL: Cannot find service/serviceDef/serviceTagsFile to use for retrieving tags. Will NOT be able to retrieve tags."); + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== init() : serviceTagsFileName=" + serviceTagsFileName); + } + } + + @Override + public ServiceTags retrieveTags(long lastKnownVersion) throws InterruptedException { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> retrieveTags(lastKnownVersion=" + lastKnownVersion + ", serviceTagsFilePath=" + serviceTagsFileName); + } + + ServiceTags serviceTags = null; + + if (serviceTagsFileURL != null) { + try { + InputStream serviceTagsFileStream = serviceTagsFileURL.openStream(); + Reader reader = new InputStreamReader(serviceTagsFileStream); + + Gson gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") + .setPrettyPrinting() + .create(); + + serviceTags = gsonBuilder.fromJson(reader, ServiceTags.class); + + if (serviceTags.getTagVersion() <= lastKnownVersion) { + // No change in serviceTags + serviceTags = null; + } + } catch (Exception e) { + LOG.warn("Error processing input file: or no privilege for reading file " + serviceTagsFileName); + } + } else { + LOG.error("Error reading file: " + serviceTagsFileName); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== retrieveTags(lastKnownVersion=" + lastKnownVersion); + } + + return serviceTags; + } + +} + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java index e9fc42c..c8fc056 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java @@ -304,15 +304,16 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { try { - populateTags(); - + // Sleep first and then fetch tags if (pollingIntervalMs > 0) { Thread.sleep(pollingIntervalMs); } else { break; } + populateTags(); + } catch (InterruptedException excp) { - LOG.info("RangerTagRefresher(pollingIntervalMs=" + pollingIntervalMs + ").run() : interrupted! Exiting thread", excp); + LOG.debug("RangerTagRefresher(pollingIntervalMs=" + pollingIntervalMs + ").run() : interrupted! Exiting thread", excp); break; } } @@ -329,7 +330,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { RangerPerfTracer perf = null; if(RangerPerfTracer.isPerfTraceEnabled(PERF_ENRICHER_LOG)) { - perf = RangerPerfTracer.getPerfTracer(PERF_ENRICHER_LOG, "RangerTagRefresher.populateTags(serviceName=" + tagRetriever.getServiceName() + ",lastKnownVersion" + lastKnownVersion + ")"); + perf = RangerPerfTracer.getPerfTracer(PERF_ENRICHER_LOG, "RangerTagRefresher.populateTags(serviceName=" + tagRetriever.getServiceName() + ",lastKnownVersion=" + lastKnownVersion + ")"); } serviceTags = tagRetriever.retrieveTags(lastKnownVersion); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java new file mode 100644 index 0000000..72da8e8 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.util; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +public class PerfDataRecorder { + private static final Log LOG = LogFactory.getLog(PerfDataRecorder.class); + private static final Log PERF = RangerPerfTracer.getPerfLogger(PerfDataRecorder.class); + + static PerfDataRecorder instance = null; + private Map<String, PerfStatistic> perfStatistics = new HashMap<String, PerfStatistic>(); + + public static void initialize(List<String> names) { + if (getPerfDataRecorder() == null) { + instance = new PerfDataRecorder(); + } + instance.init(names); + } + + public static PerfDataRecorder getPerfDataRecorder() { + return instance; + } + + public void dumpStatistics() { + for (Map.Entry<String, PerfStatistic> entry : perfStatistics.entrySet()) { + + String tag = entry.getKey(); + PerfStatistic perfStatistic = entry.getValue(); + + long averageTimeSpent = 0L; + long minTimeSpent = 0L; + long maxTimeSpent = 0L; + if (perfStatistic.numberOfInvocations.get() != 0L) { + averageTimeSpent = perfStatistic.millisecondsSpent.get()/perfStatistic.numberOfInvocations.get(); + minTimeSpent = perfStatistic.minTimeSpent.get(); + maxTimeSpent = perfStatistic.maxTimeSpent.get(); + } + + String logMsg = "[" + tag + "]" + + " execCount:" + perfStatistic.numberOfInvocations + + ", totalTimeTaken:" + perfStatistic.millisecondsSpent + + ", maxTimeTaken:" + maxTimeSpent + + ", minTimeTaken:" + minTimeSpent + + ", avgTimeTaken:" + averageTimeSpent; + + LOG.info(logMsg); + PERF.debug(logMsg); + } + } + + void record(String tag, long elapsedTime) { + PerfStatistic perfStatistic = perfStatistics.get(tag); + if (perfStatistic != null) { + perfStatistic.addPerfDataItem(elapsedTime); + } + } + + private void init(List<String> names) { + if (CollectionUtils.isNotEmpty(names)) { + for (String name : names) { + // Create structure + perfStatistics.put(name, new PerfStatistic()); + } + } + } + + private class PerfStatistic { + private AtomicLong numberOfInvocations = new AtomicLong(0L); + private AtomicLong millisecondsSpent = new AtomicLong(0L); + private AtomicLong minTimeSpent = new AtomicLong(Long.MAX_VALUE); + private AtomicLong maxTimeSpent = new AtomicLong(Long.MIN_VALUE); + + void addPerfDataItem(final long timeTaken) { + numberOfInvocations.getAndIncrement(); + millisecondsSpent.getAndAdd(timeTaken); + + long min = minTimeSpent.get(); + if (timeTaken < min) { + minTimeSpent.compareAndSet(min, timeTaken); + } + + long max = maxTimeSpent.get(); + if (timeTaken > max) { + maxTimeSpent.compareAndSet(max, timeTaken); + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java new file mode 100644 index 0000000..e7b3865 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.util; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; + +public class RangerPerfCollectorTracer extends RangerPerfTracer { + private final PerfDataRecorder recorder; + + public RangerPerfCollectorTracer(Log logger, String tag, String data, PerfDataRecorder recorder) { + super(logger, tag, data); + this.recorder = recorder; + } + + @Override + public void log() { + recorder.record(tag, getElapsedTime()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java index fc84bcd..175c4e4 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java @@ -24,10 +24,13 @@ import org.apache.commons.logging.LogFactory; import org.apache.commons.lang.StringUtils; public class RangerPerfTracer { - private final Log logger; - private final String tag; + protected final Log logger; + protected final String tag; + protected final String data; private final long startTimeMs; + private final static String tagEndMarker = "("; + public static Log getPerfLogger(String name) { return LogFactory.getLog("ranger.perf." + name); } @@ -37,15 +40,27 @@ public class RangerPerfTracer { } public static boolean isPerfTraceEnabled(Log logger) { - return logger.isInfoEnabled(); + return logger.isDebugEnabled(); } public static RangerPerfTracer getPerfTracer(Log logger, String tag) { - return logger.isInfoEnabled() ? new RangerPerfTracer(logger, tag) : null; + String data = ""; + String realTag = ""; + + if (tag != null) { + int indexOfTagEndMarker = StringUtils.indexOf(tag, tagEndMarker); + if (indexOfTagEndMarker != -1) { + realTag = StringUtils.substring(tag, 0, indexOfTagEndMarker); + data = StringUtils.substring(tag, indexOfTagEndMarker); + } else { + realTag = tag; + } + } + return RangerPerfTracerFactory.getPerfTracer(logger, realTag, data); } - public static RangerPerfTracer getPerfTracer(Log logger, Object... tagParts) { - return logger.isInfoEnabled() ? new RangerPerfTracer(logger, StringUtils.join(tagParts)) : null; + public static RangerPerfTracer getPerfTracer(Log logger, String tag, String data) { + return RangerPerfTracerFactory.getPerfTracer(logger, tag, data); } public static void log(RangerPerfTracer tracer) { @@ -54,9 +69,10 @@ public class RangerPerfTracer { } } - public RangerPerfTracer(Log logger, String tag) { + public RangerPerfTracer(Log logger, String tag, String data) { this.logger = logger; this.tag = tag; + this.data = data; startTimeMs = System.currentTimeMillis(); } @@ -73,8 +89,8 @@ public class RangerPerfTracer { } public void log() { - if(logger.isInfoEnabled()) { - logger.info("[PERF] " + tag + ": " + getElapsedTime()); + if(logger.isDebugEnabled()) { + logger.debug("[PERF] " + tag + data + ": " + getElapsedTime()); } } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java new file mode 100644 index 0000000..8db2d45 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.util; + +import org.apache.commons.logging.Log; + +public class RangerPerfTracerFactory { + + private static PerfDataRecorder perfDataRecorder = PerfDataRecorder.getPerfDataRecorder(); + + static RangerPerfTracer getPerfTracer(Log logger, String tag, String data) { + + RangerPerfTracer ret = null; + + if (perfDataRecorder != null) { + ret = new RangerPerfCollectorTracer(logger, tag, data, perfDataRecorder); + } else if (logger.isDebugEnabled()) { + ret = new RangerPerfTracer(logger, tag, data); + } + return ret; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 108bd17..23ef49e 100644 --- a/pom.xml +++ b/pom.xml @@ -107,6 +107,7 @@ <module>ranger-storm-plugin-shim</module> <module>ranger-kafka-plugin-shim</module> <module>ranger-examples</module> + <module>ranger-tools</module> </modules> <properties> <javac.source.version>1.7</javac.source.version> @@ -409,6 +410,7 @@ <descriptor>src/main/assembly/tagsync.xml</descriptor> <descriptor>src/main/assembly/migration-util.xml</descriptor> <descriptor>src/main/assembly/kms.xml</descriptor> + <descriptor>src/main/assembly/ranger-tools.xml</descriptor> <descriptor>src/main/assembly/ranger-src.xml</descriptor> </descriptors> </configuration> @@ -514,6 +516,7 @@ <exclude>**/samples/**/*.json</exclude> <exclude>**/.externalToolBuilders/*</exclude> <exclude>*.patch</exclude> + <exclude>**/testdata/*.json</exclude> <exclude>atlassian-ide-plugin.xml</exclude> <exclude>**/.pydevproject</exclude> </excludes> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/.gitignore ---------------------------------------------------------------------- diff --git a/ranger-tools/.gitignore b/ranger-tools/.gitignore new file mode 100644 index 0000000..5ac84b1 --- /dev/null +++ b/ranger-tools/.gitignore @@ -0,0 +1,2 @@ +/target/ +ranger-perftester.iml http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/conf/log4j.properties ---------------------------------------------------------------------- diff --git a/ranger-tools/conf/log4j.properties b/ranger-tools/conf/log4j.properties new file mode 100644 index 0000000..ccb9db4 --- /dev/null +++ b/ranger-tools/conf/log4j.properties @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##-- To prevent junits from cluttering the build run by default all test runs send output to null appender +log4j.appender.devnull=org.apache.log4j.varia.NullAppender +# ranger.root.logger=FATAL,devnull + +##-- uncomment the following line during during development/debugging so see debug messages during test run to be emitted to console +ranger.root.logger=INFO,console + +log4j.rootLogger=${ranger.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %L %m%n + +# +# ranger.perf log level +# +ranger.perf.logger=DEBUG,PERF +ranger.perf.log.file=ranger-perf-test.log + +log4j.logger.ranger.perf=${ranger.perf.logger} +log4j.additivity.ranger.perf=false + +log4j.appender.PERF=org.apache.log4j.DailyRollingFileAppender +log4j.appender.PERF.File=${ranger.perf.log.file} +log4j.appender.PERF.layout=org.apache.log4j.PatternLayout +log4j.appender.PERF.layout.ConversionPattern=%m%n +log4j.appender.PERF.DatePattern=.yyyy-MM-dd http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/pom.xml ---------------------------------------------------------------------- diff --git a/ranger-tools/pom.xml b/ranger-tools/pom.xml new file mode 100644 index 0000000..f7347f7 --- /dev/null +++ b/ranger-tools/pom.xml @@ -0,0 +1,69 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>ranger</artifactId> + <groupId>org.apache.ranger</groupId> + <version>0.5.0</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>ranger-tools</artifactId> + <packaging>jar</packaging> + + <name>Ranger Tools</name> + <url>http://maven.apache.org</url> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + </properties> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>${junit.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>commons-cli</groupId> + <artifactId>commons-cli</artifactId> + <version>1.3.1</version> + </dependency> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + <version>${commons.logging.version}</version> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>${commons.lang.version}</version> + </dependency> + <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + </dependency> + <dependency> + <groupId>security_plugins.ranger-plugins-common</groupId> + <artifactId>ranger-plugins-common</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/scripts/README.txt ---------------------------------------------------------------------- diff --git a/ranger-tools/scripts/README.txt b/ranger-tools/scripts/README.txt new file mode 100644 index 0000000..9eb71ba --- /dev/null +++ b/ranger-tools/scripts/README.txt @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +This file describes how to build, unpackage and run the performance testing tool. + +1. Build Apache Ranger using the following command. + % mvn clean compile package assembly:assembly + + The following artifact will be created under target directory. + + target/ranger-0.5.0-ranger-tools.tar.gz + +2. Copy this artifact to the directory where you want to run the tool. + + % cp target/ranger-0.5.0-ranger-tools.tar.gz <perf-tool-run-dir> + % cd <perf-tool-run-dir> + +3. Unzip the artifact. + + % tar xvfz ranger-0.5.0-ranger-tools.tar.gz + + This will create the following directory structure under <perf-tool-run-dir> + + ranger-0.5.0-ranger-tools + ranger-0.5.0-ranger-tools/conf + ranger-0.5.0-ranger-tools/dist + ranger-0.5.0-ranger-tools/lib + ranger-0.5.0-ranger-tools/scripts + ranger-0.5.0-ranger-tools/testdata + +4. % cd ranger-0.5.0-ranger-tools + +5. Setup configuration + + Following sample data files are packaged with the perf-tool: + service-policies - testdata/test_servicepolicies_hive.json + requests - testdata/test_requests_hive.json + modules-to-profile - testdata/test_modules.txt + service-tags - testdata/test_servicetags_hive.json (used only for tag-based policies; referenced from service-policies file. + + Please review the contents of these files and modify (or copy/modify) to suite your policy and request needs. + + Update conf/log4j.properties to specify the filename where perf run results will be written to. Property to update is 'ranger.perf.logger'. + +5. Run the tool with the command, + + % ./ranger-perftester.sh -s <service-policies-file> -r <requests-file> -p <profiled-modules-file> -c <number-of-concurrent-clients> -n <number-of-times-requests-file-to-be-run> + + Example: + % ./ranger-perftester.sh -s testdata/test_servicepolicies_hive.json -r testdata/test_requests_hive.json -p testdata/test_modules.txt -c 2 -n 1 + +6. At the end of the run, the performance-statistics are printed in the log file in conf/log4j.properties file. + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/scripts/ranger-perftester.sh ---------------------------------------------------------------------- diff --git a/ranger-tools/scripts/ranger-perftester.sh b/ranger-tools/scripts/ranger-perftester.sh new file mode 100755 index 0000000..46c8e0e --- /dev/null +++ b/ranger-tools/scripts/ranger-perftester.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cdir=$(cd "$(dirname "$0")"; pwd) +cp="${cdir}/dist/*:${cdir}/lib/*:${cdir}/conf:." + +if [ "${JAVA_HOME}" != "" ] +then + export JAVA_HOME + PATH="${JAVA_HOME}/bin:${PATH}" + export PATH +fi + +JAVA_CMD="java -cp ${cp} org.apache.ranger.policyengine.RangerPolicyenginePerfTester" + +cd ${cdir} + +echo "JAVA command = $JAVA_CMD " "$@" +$JAVA_CMD "$@" http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/scripts/summary.awk ---------------------------------------------------------------------- diff --git a/ranger-tools/scripts/summary.awk b/ranger-tools/scripts/summary.awk new file mode 100755 index 0000000..16e49f3 --- /dev/null +++ b/ranger-tools/scripts/summary.awk @@ -0,0 +1,17 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +awk '{ORS=" "; print $NF; for (i=6; i<NF; i++) print $i;print "\n" }' $1 | sort -n -r http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java new file mode 100644 index 0000000..a45d71a --- /dev/null +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.policyengine; + +import org.apache.commons.cli.*; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Arrays; + +public class CommandLineParser +{ + static final Log LOG = LogFactory.getLog(CommandLineParser.class); + + private String servicePoliciesFileName; + private String[] requestFileNames; + private String statCollectionFileName; + + private URL servicePoliciesFileURL; + private URL[] requestFileURLs; + private URL statCollectionFileURL; + + + private int concurrentClientCount = 1; + private int iterationsCount = 1; + + private Options options = new Options(); + + CommandLineParser() {} + + final PerfTestOptions parse(final String[] args) { + PerfTestOptions ret = null; + if (parseArguments(args) && validateInputFiles()) { + // Instantiate a data-object and return + ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount); + } else { + showUsage(-1); + } + return ret; + } + + // Parse the arguments + + /* Arguments : + -s servicePolicies-file-name + -c concurrent-client-count + -r request-file-name-list + -n number-of-iterations + -p modules-to-collect-stats + + If the concurrent-client-count is more than the number of files in the request-file-name-list, + then reuse the request-file-names in a round-robin way + + */ + + final boolean parseArguments(final String[] args) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> parseArguments()"); + } + boolean ret = false; + + options.addOption("h", "help", false, "show help."); + options.addOption("s", "service-policies", true, "Policies File Name"); + options.addOption("r", "requests", true, "Request Definition File Name"); + options.addOption("p", "statistics", true, "Modules for stat collection File Name"); + options.addOption("c", "clients", true, "Number of concurrent clients"); + options.addOption("n", "cycles", true, "Number of iterations"); + + org.apache.commons.cli.CommandLineParser commandLineParser = new DefaultParser(); + + try { + CommandLine commandLine = commandLineParser.parse(options, args); + + if (commandLine.hasOption("h")) { + showUsage(0); + } + + servicePoliciesFileName = commandLine.getOptionValue("s"); + requestFileNames = commandLine.getOptionValues("r"); + statCollectionFileName = commandLine.getOptionValue("p"); + + concurrentClientCount = 1; + String clientOptionValue = commandLine.getOptionValue("c"); + if (clientOptionValue != null) { + concurrentClientCount = Integer.parseInt(clientOptionValue); + } + + iterationsCount = 1; + String iterationsOptionValue = commandLine.getOptionValue("n"); + if (iterationsOptionValue != null) { + iterationsCount = Integer.parseInt(iterationsOptionValue); + } + if (LOG.isDebugEnabled()) { + LOG.debug("servicePoliciesFileName=" + servicePoliciesFileName + ", requestFileName=" + Arrays.toString(requestFileNames)); + LOG.debug("concurrentClientCount=" + concurrentClientCount + ", iterationsCount=" + iterationsCount); + } + + ret = true; + } catch (Exception exception) { + LOG.error("Error processing command-line arguments: ", exception); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== parseArguments() : " + ret); + } + + return ret; + } + + final boolean validateInputFiles() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> validateInputFiles()"); + } + + boolean ret = false; + + if (servicePoliciesFileName != null) { + this.servicePoliciesFileURL = getInputFileURL(servicePoliciesFileName); + if (servicePoliciesFileURL != null) { + if (requestFileNames != null) { + if (validateRequestFiles()) { + if (statCollectionFileName != null) { + statCollectionFileURL = getInputFileURL(statCollectionFileName); + ret = statCollectionFileURL != null; + } else { + LOG.error("Error processing stat-collection-module file"); + } + } + } else { + LOG.error("Error processing requests file: No requests files provided."); + } + } else { + LOG.error("Error processing service-policies file: unreadable service-policies file: " + servicePoliciesFileName); + } + } else { + LOG.error("Error processing service-policies file: null service-policies file"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== validateInputFiles(): " + ret); + } + + return ret; + } + + final boolean validateRequestFiles() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> validateRequestFiles()"); + } + boolean ret = requestFileNames.length > 0; + + if (ret) { + requestFileURLs = new URL[requestFileNames.length]; + + for (int i = 0; ret && i < requestFileNames.length; i++) { + if (requestFileNames[i] != null) { + if ((requestFileURLs[i] = getInputFileURL(requestFileNames[i])) == null) { + LOG.error("Cannot read file: " + requestFileNames[i]); + ret = false; + } + } else { + LOG.error("Error processing request-file: null input file-name for request-file"); + ret = false; + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== validateRequestFiles(): " + ret); + } + return ret; + } + + public static URL getInputFileURL(final String name) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> getResourceFileURL(" + name + ")"); + } + URL ret = null; + InputStream in = null; + + + if (StringUtils.isNotBlank(name)) { + + File f = new File(name); + + if (f.exists() && f.isFile() && f.canRead()) { + try { + + in = new FileInputStream(f); + ret = f.toURI().toURL(); + if (LOG.isDebugEnabled()) { + LOG.debug("URL:" + ret); + } + + } catch (FileNotFoundException exception) { + LOG.error("Error processing input file:" + name + " or no privilege for reading file " + name, exception); + } catch (MalformedURLException malformedException) { + LOG.error("Error processing input file:" + name + " cannot be converted to URL " + name, malformedException); + } + } else { + + URL fileURL = CommandLineParser.class.getResource(name); + if (fileURL == null) { + if (!name.startsWith("/")) { + fileURL = CommandLineParser.class.getResource("/" + name); + } + } + + if (fileURL == null) { + fileURL = ClassLoader.getSystemClassLoader().getResource(name); + if (fileURL == null) { + if (!name.startsWith("/")) { + fileURL = ClassLoader.getSystemClassLoader().getResource("/" + name); + } + } + } + + if (fileURL != null) { + try { + in = fileURL.openStream(); + ret = fileURL; + } catch (Exception exception) { + LOG.error(name + " cannot be opened:", exception); + } + } else { + LOG.warn("Error processing input file: URL not found for " + name + " or no privilege for reading file " + name); + } + } + } + if (in != null) { + try { + in.close(); + } catch (Exception e) { + // Ignore + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== getResourceFileURL(" + name + ", URL=" + ret + ")"); + } + return ret; + } + + void showUsage(int exitCode) { + HelpFormatter formater = new HelpFormatter(); + formater.printHelp("perfTester", options); + + LOG.info("Exiting..."); + + System.exit(exitCode); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java new file mode 100644 index 0000000..b88d670 --- /dev/null +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.policyengine; + +import com.google.gson.*; +import com.google.gson.reflect.TypeToken; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.policyengine.*; + +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.lang.reflect.Type; +import java.net.URL; +import java.util.List; + +public class PerfTestClient extends Thread { + static final Log LOG = LogFactory.getLog(PerfTestClient.class); + + final PerfTestEngine perfTestEngine; + final int clientId; + final URL requestFileURL; + final int maxCycles; + + List<RequestData> requests = null; + static Gson gsonBuilder = null; + + static { + + gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") + .setPrettyPrinting() + .registerTypeAdapter(RangerAccessRequest.class, new RangerAccessRequestDeserializer()) + .registerTypeAdapter(RangerAccessResource.class, new RangerResourceDeserializer()) + .create(); + } + + public PerfTestClient(final PerfTestEngine perfTestEngine, final int clientId, final URL requestFileURL, final int maxCycles) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> PerfTestClient(clientId=" + clientId + ", maxCycles=" + maxCycles +")" ); + } + + this.perfTestEngine = perfTestEngine; + this.clientId = clientId; + this.requestFileURL = requestFileURL; + this.maxCycles = maxCycles; + + setName("PerfTestClient-" + clientId); + setDaemon(true); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== PerfTestClient(clientId=" + clientId + ", maxCycles=" + maxCycles +")" ); + } + } + + public boolean init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> init()" ); + } + + boolean ret = false; + + Reader reader = null; + + try { + + InputStream in = requestFileURL.openStream(); + + reader = new InputStreamReader(in); + + Type listType = new TypeToken<List<RequestData>>() { + }.getType(); + + requests = gsonBuilder.fromJson(reader, listType); + + ret = true; + } + catch (Exception excp) { + LOG.error("Error opening request data stream or loading load request data from file, URL=" + requestFileURL, excp); + } + finally { + if (reader != null) { + try { + reader.close(); + } catch (Exception excp) { + LOG.error("Error closing file ", excp); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== init() : " + ret ); + } + return ret; + } + + @Override + public void run() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> run()" ); + } + + try { + for (int i = 0; i < maxCycles; i++) { + for (RequestData data : requests) { + perfTestEngine.execute(data.request); + } + } + } catch (Exception excp) { + LOG.error("PerfTestClient.run() : interrupted! Exiting thread", excp); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== run()" ); + } + } + + private class RequestData { + public String name; + public RangerAccessRequest request; + public RangerAccessResult result; + } + + static class RangerAccessRequestDeserializer implements JsonDeserializer<RangerAccessRequest> { + @Override + public RangerAccessRequest deserialize(JsonElement jsonObj, Type type, + JsonDeserializationContext context) throws JsonParseException { + RangerAccessRequestImpl ret = gsonBuilder.fromJson(jsonObj, RangerAccessRequestImpl.class); + + ret.setAccessType(ret.getAccessType()); // to force computation of isAccessTypeAny and isAccessTypeDelegatedAdmin + + return ret; + } + } + + static class RangerResourceDeserializer implements JsonDeserializer<RangerAccessResource> { + @Override + public RangerAccessResource deserialize(JsonElement jsonObj, Type type, + JsonDeserializationContext context) throws JsonParseException { + return gsonBuilder.fromJson(jsonObj, RangerAccessResourceImpl.class); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java new file mode 100644 index 0000000..1b5f6fd --- /dev/null +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.policyengine; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler; +import org.apache.ranger.plugin.policyengine.*; +import org.apache.ranger.plugin.util.ServicePolicies; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.URL; + +public class PerfTestEngine { + static final Log LOG = LogFactory.getLog(PerfTestEngine.class); + + private final URL servicePoliciesFileURL; + private RangerPolicyEngine policyEvaluationEngine; + + public PerfTestEngine(final URL servicePoliciesFileURL) { + this.servicePoliciesFileURL = servicePoliciesFileURL; + } + + public boolean init() { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> init()"); + } + + boolean ret = false; + + Gson gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") + .setPrettyPrinting() + .create(); + + Reader reader = null; + ServicePolicies servicePolicies; + + try { + InputStream in = servicePoliciesFileURL.openStream(); + + reader = new InputStreamReader(in); + + servicePolicies = gsonBuilder.fromJson(reader, ServicePolicies.class); + + RangerPolicyEngineOptions engineOptions = new RangerPolicyEngineOptions(); + engineOptions.disableTagPolicyEvaluation = false; + + policyEvaluationEngine = new RangerPolicyEngineImpl("perf-test", servicePolicies, engineOptions); + + ret = true; + + } catch (Exception excp) { + LOG.error("Error opening service-policies file or loading service-policies from file, URL=" + servicePoliciesFileURL, excp); + } finally { + if (reader != null) { + try { + reader.close(); + } catch (Exception excp) { + LOG.error("Error closing file", excp); + } + } + } + + + if (LOG.isDebugEnabled()) { + LOG.debug("<== init() : " + ret); + } + + return ret; + + } + public boolean execute(final RangerAccessRequest request) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> execute(" + request + ")"); + } + + boolean ret = true; + + if (policyEvaluationEngine != null) { + + RangerAccessResultProcessor auditHandler = null; + + policyEvaluationEngine.preProcess(request); + + RangerAccessResult result = policyEvaluationEngine.isAccessAllowed(request, auditHandler); + } else { + LOG.error("Error executing request: PolicyEngine is null!"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== execute(" + request + ") : " + ret); + } + + return ret; + } + + public void cleanup() { + if (policyEvaluationEngine != null) { + policyEvaluationEngine.cleanup(); + policyEvaluationEngine = null; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java new file mode 100644 index 0000000..f30cbd7 --- /dev/null +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.policyengine; + + +import java.net.URL; + +public class PerfTestOptions { + private final URL servicePoliciesFileURL; + private final URL[] requestFileURLs; + private final URL statCollectionFileURL; + + + private final int concurrentClientCount; + private final int iterationsCount; + + PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount) { + this.servicePoliciesFileURL = servicePoliciesFileURL; + this.requestFileURLs = requestFileURLs; + this.statCollectionFileURL = statCollectionFileURL; + this.iterationsCount = iterationsCount; + this.concurrentClientCount = concurrentClientCount; + } + + public URL getServicePoliciesFileURL() { + return this.servicePoliciesFileURL; + } + + public URL[] getRequestFileURLs() { + return this.requestFileURLs; + } + + public URL getStatCollectionFileURL() { + return this.statCollectionFileURL; + } + + public int getConcurrentClientCount() { + return concurrentClientCount; + } + + public int getIterationsCount() { + return iterationsCount; + }} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java new file mode 100644 index 0000000..28cc558 --- /dev/null +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.policyengine; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.util.PerfDataRecorder; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; + +public class RangerPolicyenginePerfTester { + static final Log LOG = LogFactory.getLog(RangerPolicyenginePerfTester.class); + + public static void main(String[] args) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyenginePerfTester.main()"); + } + + CommandLineParser commandLineParser = new CommandLineParser(); + + PerfTestOptions perfTestOptions = commandLineParser.parse(args); + + URL statCollectionFileURL = perfTestOptions.getStatCollectionFileURL(); + + List<String> perfModuleNames = buildPerfModuleNames(statCollectionFileURL); + + PerfDataRecorder.initialize(perfModuleNames); + + URL servicePoliciesFileURL = perfTestOptions.getServicePoliciesFileURL(); + + PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL); + if (!perfTestEngine.init()) { + LOG.error("Error initializing test data. Existing..."); + System.exit(1); + } + + URL[] requestFileURLs = perfTestOptions.getRequestFileURLs(); + int requestFilesCount = requestFileURLs.length; + + int clientsCount = perfTestOptions.getConcurrentClientCount(); + List<PerfTestClient> perfTestClients = new ArrayList<PerfTestClient>(clientsCount); + + for (int i = 0; i < clientsCount; i++) { + + URL requestFileURL = requestFileURLs[i % requestFilesCount]; + + PerfTestClient perfTestClient = new PerfTestClient(perfTestEngine, i, requestFileURL, perfTestOptions.getIterationsCount()); + + if (!perfTestClient.init()) { + LOG.error("Error initializing PerfTestClient: (id=" + i + ")"); + } else { + perfTestClients.add(perfTestClient); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Number of perfTestClients=" + perfTestClients.size()); + } + + for (PerfTestClient client : perfTestClients) { + try { + client.start(); + } catch (Throwable t) { + LOG.error("Error in starting client: " + client.getName(), t); + } + } + + LOG.info("Waiting for " + perfTestClients.size() + " clients to finish up"); + + for (PerfTestClient client : perfTestClients) { + try { + if (client.isAlive()) { + LOG.info("Waiting for " + client.getName() + " to finish up."); + client.join(); + } + } catch (InterruptedException interruptedException) { + LOG.error("PerfTestClient.join() was interrupted"); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyenginePerfTester.main()"); + } + + LOG.info("Completed performance-run"); + + perfTestEngine.cleanup(); + + PerfDataRecorder.getPerfDataRecorder().dumpStatistics(); + } + + private static List<String> buildPerfModuleNames(URL statCollectionFileURL) { + List<String> perfModuleNames = new ArrayList<String>(); + + try ( + InputStream inStream = statCollectionFileURL.openStream(); + InputStreamReader reader = new InputStreamReader(inStream, Charset.forName("UTF-8")); + BufferedReader br = new BufferedReader(reader); + ) { + + String line; + + while ((line = br.readLine()) != null) { + line = line.trim(); + if (!line.isEmpty() && !line.startsWith("#")) { + String[] moduleNames = line.split(" "); + for (int i = 0; i < moduleNames.length; i++) { + perfModuleNames.add(moduleNames[i]); + } + } + } + } catch (Exception exception) { + System.out.println("Error reading arguments:" + exception); + } + + return perfModuleNames; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/test/java/org/apache/ranger/policyengine/PerfTesterTest.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/test/java/org/apache/ranger/policyengine/PerfTesterTest.java b/ranger-tools/src/test/java/org/apache/ranger/policyengine/PerfTesterTest.java new file mode 100644 index 0000000..2d7c52e --- /dev/null +++ b/ranger-tools/src/test/java/org/apache/ranger/policyengine/PerfTesterTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.policyengine; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.Charset; + +/** + * Unit test for simple App. + */ +public class PerfTesterTest + extends TestCase + +{ + /** + * Create the test case + * + * @param testName name of the test case + */ + public PerfTesterTest( String testName ) + { + super( testName ); + } + + /** + * @return the suite of tests being tested + */ + public static Test suite() + { + return new TestSuite( PerfTesterTest.class ); + } + + /** + * Rigourous Test :-) + */ + + public void testMain() { + + String[] args = readCommandLine(); + + if (args != null) { + RangerPolicyenginePerfTester.main(args); + } + } + + + public void testArgParsing() { + String[] args = readCommandLine(); + + if (args != null) { + CommandLineParser commandLineParser = new CommandLineParser(); + PerfTestOptions parseResult = commandLineParser.parse(args); + assertNotNull(parseResult); + } + } + + String[] readCommandLine() { + // Read arguments from a file - with hardcoded name 'commandline' + + String[] ret = null; + + URL commandLineFileURL = CommandLineParser.getInputFileURL("/commandline"); + if (commandLineFileURL != null) { + try ( + InputStream inStream = commandLineFileURL.openStream(); + InputStreamReader reader = new InputStreamReader(inStream, Charset.forName("UTF-8")); + BufferedReader br = new BufferedReader(reader); + ) { + + + String line; + + while ((line = br.readLine()) != null) { + line = line.trim(); + if (!line.isEmpty() && !line.startsWith("#")) { + ret = line.split(" "); + break; + } + } + + } catch (Exception exception) { + System.out.println("Error reading arguments:" + exception); + } + } + return ret; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/test/resources/commandline ---------------------------------------------------------------------- diff --git a/ranger-tools/src/test/resources/commandline b/ranger-tools/src/test/resources/commandline new file mode 100644 index 0000000..9ea690e --- /dev/null +++ b/ranger-tools/src/test/resources/commandline @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +-s /testdata/test_servicepolicies_hive.json -r /testdata/test_requests_hive.json -p /testdata/test_modules.txt -c 3 -n 1 http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/ranger-tools/src/test/resources/log4j.properties b/ranger-tools/src/test/resources/log4j.properties new file mode 100644 index 0000000..a9a8881 --- /dev/null +++ b/ranger-tools/src/test/resources/log4j.properties @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##-- To prevent junits from cluttering the build run by default all test runs send output to null appender +log4j.appender.devnull=org.apache.log4j.varia.NullAppender +# ranger.root.logger=FATAL,devnull + +##-- uncomment the following line during during development/debugging so see debug messages during test run to be emitted to console +ranger.root.logger=INFO,console + +log4j.rootLogger=${ranger.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %L %m%n + +# +# ranger.perf log level +# +ranger.perf.logger=DEBUG,PERF +ranger.perf.log.file=/tmp/ranger-perf-test.log + +log4j.logger.ranger.perf=${ranger.perf.logger} +log4j.additivity.ranger.perf=false + +log4j.appender.PERF=org.apache.log4j.DailyRollingFileAppender +log4j.appender.PERF.File=${ranger.perf.log.file} +log4j.appender.PERF.layout=org.apache.log4j.PatternLayout +log4j.appender.PERF.layout.ConversionPattern=%m%n +log4j.appender.PERF.DatePattern=.yyyy-MM-dd http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/test/resources/testdata/test_modules.txt ---------------------------------------------------------------------- diff --git a/ranger-tools/src/test/resources/testdata/test_modules.txt b/ranger-tools/src/test/resources/testdata/test_modules.txt new file mode 100644 index 0000000..8eb5746 --- /dev/null +++ b/ranger-tools/src/test/resources/testdata/test_modules.txt @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +RangerTagEnricher.setServiceTags +RangerPolicyEngine.init +RangerPolicyEngine.preProcess +RangerPolicyEngine.isAccessAllowedNoAudit + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/test/resources/testdata/test_requests_hive.json ---------------------------------------------------------------------- diff --git a/ranger-tools/src/test/resources/testdata/test_requests_hive.json b/ranger-tools/src/test/resources/testdata/test_requests_hive.json new file mode 100644 index 0000000..0db7207 --- /dev/null +++ b/ranger-tools/src/test/resources/testdata/test_requests_hive.json @@ -0,0 +1,10 @@ + [ + {"name":"'select default/tbl-0/col-2;' for hrt_1", + "request":{ + "resource":{"elements":{"database":"default","table":"tbl-0","column":"col-2"}}, + "accessType":"select","user":"hrt_1","userGroups":[],"requestData":"use default" + }, + "result":{"isAudited":true,"isAllowed":true,"policyId":2} + } + ] + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/test/resources/testdata/test_servicepolicies_hive.json ---------------------------------------------------------------------- diff --git a/ranger-tools/src/test/resources/testdata/test_servicepolicies_hive.json b/ranger-tools/src/test/resources/testdata/test_servicepolicies_hive.json new file mode 100644 index 0000000..4d3a166 --- /dev/null +++ b/ranger-tools/src/test/resources/testdata/test_servicepolicies_hive.json @@ -0,0 +1,488 @@ +{ + "serviceName": "cl1_hive", + "serviceId": 2, + "policies": [ + { + "service": "cl1_hive", + "name": "cl1_hive-1-20151212014502", + "isAuditEnabled": false, + "resources": { + "database": { + "values": [ + "*" + ], + "isExcludes": false, + "isRecursive": false + }, + "column": { + "values": [ + "*" + ], + "isExcludes": false, + "isRecursive": false + }, + "table": { + "values": [ + "*" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "select", + "isAllowed": true + }, + { + "type": "update", + "isAllowed": true + }, + { + "type": "create", + "isAllowed": true + }, + { + "type": "drop", + "isAllowed": true + }, + { + "type": "alter", + "isAllowed": true + }, + { + "type": "index", + "isAllowed": true + }, + { + "type": "lock", + "isAllowed": true + }, + { + "type": "all", + "isAllowed": true + } + ], + "users": [ + "ambari-qa" + ], + "groups": [], + "conditions": [], + "delegateAdmin": true, + "isEnabled": true + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "id": 2, + "isEnabled": true + } + ], + "serviceDef": { + "name": "hive", + "implClass": "org.apache.ranger.services.hive.RangerServiceHive", + "label": "Hive Server2", + "options": {}, + "configs": [ + { + "itemId": 1, + "name": "username", + "type": "string", + "mandatory": true, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Username" + }, + { + "itemId": 2, + "name": "password", + "type": "password", + "mandatory": true, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Password" + }, + { + "itemId": 3, + "name": "jdbc.driverClassName", + "type": "string", + "mandatory": true, + "defaultValue": "org.apache.hive.jdbc.HiveDriver", + "validationRegEx": "", + "validationMessage": "", + "uiHint": "" + }, + { + "itemId": 4, + "name": "jdbc.url", + "type": "string", + "mandatory": true, + "defaultValue": "", + "validationRegEx": "", + "validationMessage": "", + "uiHint": "" + }, + { + "itemId": 5, + "name": "commonNameForCertificate", + "type": "string", + "mandatory": false, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Common Name for Certificate" + } + ], + "resources": [ + { + "itemId": 1, + "name": "database", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Hive Database" + }, + { + "itemId": 2, + "name": "table", + "type": "string", + "level": 20, + "parent": "database", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Hive Table" + }, + { + "itemId": 3, + "name": "udf", + "type": "string", + "level": 20, + "parent": "database", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Hive UDF" + }, + { + "itemId": 4, + "name": "column", + "type": "string", + "level": 30, + "parent": "table", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Hive Column" + } + ], + "accessTypes": [ + { + "itemId": 1, + "name": "select", + "label": "select", + "impliedGrants": [] + }, + { + "itemId": 2, + "name": "update", + "label": "update", + "impliedGrants": [] + }, + { + "itemId": 3, + "name": "create", + "label": "Create", + "impliedGrants": [] + }, + { + "itemId": 4, + "name": "drop", + "label": "Drop", + "impliedGrants": [] + }, + { + "itemId": 5, + "name": "alter", + "label": "Alter", + "impliedGrants": [] + }, + { + "itemId": 6, + "name": "index", + "label": "Index", + "impliedGrants": [] + }, + { + "itemId": 7, + "name": "lock", + "label": "Lock", + "impliedGrants": [] + }, + { + "itemId": 8, + "name": "all", + "label": "All", + "impliedGrants": [ + "select", + "update", + "create", + "drop", + "alter", + "index", + "lock" + ] + } + ], + "policyConditions": [ + { + "itemId": 1, + "name": "resources-accessed-together", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerHiveResourcesAccessedTogetherCondition", + "evaluatorOptions": {}, + "label": "Hive Resources Accessed Together?" + } + ], + "contextEnrichers": [], + "enums": [], + "id": 3, + "isEnabled": true + }, + "tagPolicies": { + "serviceName": "tagdev", + "serviceId": 3, + "policyVersion": 1, + "policyUpdateTime": "20151212-01:48:43.000-+0000", + "policies": [ + { + "service": "tagdev", + "name": "tagdev-EXPIRES_ON", + "isAuditEnabled": true, + "resources": { + "tag": { + "values": [ + "EXPIRES_ON" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [], + "denyPolicyItems": [ + { + "accesses": [ + { + "type": "hive:select", + "isAllowed": true + }, + { + "type": "hive:update", + "isAllowed": true + }, + { + "type": "hive:create", + "isAllowed": true + }, + { + "type": "hive:drop", + "isAllowed": true + }, + { + "type": "hive:alter", + "isAllowed": true + }, + { + "type": "hive:index", + "isAllowed": true + }, + { + "type": "hive:lock", + "isAllowed": true + }, + { + "type": "hive:all", + "isAllowed": true + } + ], + "users": [], + "groups": [ + "public" + ], + "conditions": [ + { + "type": "accessed-after-expiry", + "values": [ + "yes" + ] + } + ], + "delegateAdmin": true, + "isEnabled": true + } + ], + "allowExceptions": [], + "denyExceptions": [], + "id": 4, + "isEnabled": true + } + ], + "serviceDef": { + "name": "tag", + "implClass": "org.apache.ranger.services.tag.RangerServiceTag", + "label": "TAG", + "options": { + "ui.pages": "tag-based-policies" + }, + "configs": [], + "resources": [ + { + "itemId": 1, + "name": "tag", + "type": "string", + "level": 1, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "false", + "ignoreCase": "false" + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "{ \"singleValue\":true }", + "label": "TAG" + } + ], + "accessTypes": [ + { + "itemId": 3004, + "name": "hive:select", + "label": "select", + "impliedGrants": [] + }, + { + "itemId": 3005, + "name": "hive:update", + "label": "update", + "impliedGrants": [] + }, + { + "itemId": 3006, + "name": "hive:create", + "label": "Create", + "impliedGrants": [] + }, + { + "itemId": 3007, + "name": "hive:drop", + "label": "Drop", + "impliedGrants": [] + }, + { + "itemId": 3008, + "name": "hive:alter", + "label": "Alter", + "impliedGrants": [] + }, + { + "itemId": 3009, + "name": "hive:index", + "label": "Index", + "impliedGrants": [] + }, + { + "itemId": 3010, + "name": "hive:lock", + "label": "Lock", + "impliedGrants": [] + }, + { + "itemId": 3011, + "name": "hive:all", + "label": "All", + "impliedGrants": [ + "hive:select", + "hive:update", + "hive:create", + "hive:drop", + "hive:alter", + "hive:index", + "hive:lock" + ] + } + ], + "policyConditions": [ + { + "itemId": 1, + "name": "accessed-after-expiry", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptTemplateConditionEvaluator", + "evaluatorOptions": { + "scriptTemplate": "ctx.isAccessedAfter(\u0027expiry_date\u0027);" + }, + "uiHint": "{ \"singleValue\":true }", + "label": "Accessed after expiry_date (yes/no)?" + } + ], + "contextEnrichers": [ + { + "itemId": 1, + "name": "TagEnricher", + "enricher": "org.apache.ranger.plugin.contextenricher.RangerTagEnricher", + "enricherOptions": { + "tagRetrieverClassName": "org.apache.ranger.plugin.contextenricher.RangerFileBasedTagRetriever", + "tagRefresherPollingInterval": "60000", + "serviceTagsFileName":"/testdata/test_servicetags_hive.json" + } + } + ], + "enums": [], + "id": 100, + "isEnabled": true + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/test/resources/testdata/test_servicetags_hive.json ---------------------------------------------------------------------- diff --git a/ranger-tools/src/test/resources/testdata/test_servicetags_hive.json b/ranger-tools/src/test/resources/testdata/test_servicetags_hive.json new file mode 100644 index 0000000..73c9264 --- /dev/null +++ b/ranger-tools/src/test/resources/testdata/test_servicetags_hive.json @@ -0,0 +1,62 @@ +{ + "op": "add_or_update", + "tagModel": "shared", + "serviceName": "cl1_hive", + "tagVersion": 500, + "tagDefinitions": { + "1": { + "name": "EXPIRES_ON_1", + "source": "Internal", + "attributeDefs": [ + { + "name": "activates_on", + "type": "datetime" + }, + { + "name": "expiry_date", + "type": "datetime" + } + ], + "id": 1, + "isEnabled": true + } + }, + "tags": { + "1": { + "type": "EXPIRES_ON_1", + "attributes": { + "expiry_date": "2017/12/31", + "activates_on": "2017/01/01" + }, + "id": 1, + "isEnabled": true + } + }, + "serviceResources": [ + { + "resourceElements": { + "database": { + "values": [ + "finance_1" + ], + "isExcludes": false, + "isRecursive": false + }, + "table": { + "values": [ + "tax_2010_1" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "id": 1, + "isEnabled": true + } + ], + "resourceToTagIds": { + "1": [ + 1 + ] + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/testdata/test_modules.txt ---------------------------------------------------------------------- diff --git a/ranger-tools/testdata/test_modules.txt b/ranger-tools/testdata/test_modules.txt new file mode 100644 index 0000000..03771fa --- /dev/null +++ b/ranger-tools/testdata/test_modules.txt @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +RangerTagEnricher.setServiceTags +RangerPolicyEngine.init +RangerPolicyEngine.preProcess +RangerPolicyEngine.isAccessAllowedNoAudit
